You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mxnet.apache.org by cj...@apache.org on 2018/03/14 17:21:27 UTC
[incubator-mxnet] 01/10: Refreshed branch cython
This is an automated email from the ASF dual-hosted git repository.
cjolivier01 pushed a commit to branch cython
in repository https://gitbox.apache.org/repos/asf/incubator-mxnet.git
commit de0db8ff8400c0dd968a6609e362747b931d7b0e
Author: Olivier <co...@amazon.com>
AuthorDate: Thu Mar 8 08:46:46 2018 -0800
Refreshed branch cython
---
CMakeLists.txt | 67 +++-
cmake/CythonUtil.cmake | 297 ++++++++++++++++
cmake/Modules/FindCython.cmake | 69 ++++
cmake/Modules/FindOpenBLAS.cmake | 9 +-
cmake/UseCython.cmake | 390 +++++++++++++++++++++
python/mxnet/{_cy3 => cython}/__init__.py | 2 -
python/mxnet/cython/base.pyi | 2 +-
python/mxnet/{_cy3 => cython/cy2}/README | 0
python/mxnet/{_cy2 => cython/cy2}/__init__.py | 0
python/mxnet/{_cy2 => cython/cy3}/README | 0
python/mxnet/{_cy3 => cython/cy3}/__init__.py | 0
python/mxnet/cython/mxcython.pyx | 86 +++++
python/mxnet/{_cy2 => ndarray/cy2}/__init__.py | 0
python/mxnet/{_cy3 => ndarray/cy3}/__init__.py | 0
python/mxnet/{cython => ndarray}/ndarray.pyx | 4 +-
python/mxnet/{_cy2 => symbol/cy2}/__init__.py | 0
python/mxnet/{_cy3 => symbol/cy3}/__init__.py | 0
python/mxnet/{cython => symbol}/symbol.pyx | 8 +-
python/setup.py | 5 +-
.../__init__.py => src/cython/cy2/CMakeLists.txt | 17 +-
.../__init__.py => src/cython/cy3/CMakeLists.txt | 17 +-
src/cython/cython_util.cc | 103 ++++++
src/cython/cython_util.h | 51 +++
tests/python/unittest/test_cython.py | 75 ++++
.../__init__.py => tools/cython/clean_cython.sh | 6 +-
.../mxnet/_cy3/__init__.py => tools/cython/cydb2 | 11 +-
.../mxnet/_cy3/__init__.py => tools/cython/cydb3 | 11 +-
27 files changed, 1198 insertions(+), 32 deletions(-)
diff --git a/CMakeLists.txt b/CMakeLists.txt
index b3a8955..04f0795 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -1,3 +1,20 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
cmake_minimum_required(VERSION 3.0.2)
project(mxnet C CXX)
@@ -36,6 +53,7 @@ mxnet_option(ENABLE_CUDA_RTC "Build with CUDA runtime compilation support"
mxnet_option(BUILD_CPP_EXAMPLES "Build cpp examples" ON)
mxnet_option(INSTALL_EXAMPLES "Install the example source files." OFF)
mxnet_option(USE_SIGNAL_HANDLER "Print stack traces on segfaults." OFF)
+mxnet_option(INSTALL_CYTHON_INPLACE "Install cython modules into source python tree." ON)
if(USE_CUDA AND NOT USE_OLDCMAKECUDA)
message(STATUS "CMake version '${CMAKE_VERSION}' using generator '${CMAKE_GENERATOR}'")
@@ -539,14 +557,14 @@ if(USE_CUDA)
add_definitions(-DMXNET_USE_CUDA=1)
if(CUDA_LIBRARY_PATH)
if(IS_CONTAINER_BUILD)
- # In case of building on a production-like build container which may not have Cuda installed
- if(NOT CMAKE_SYSTEM_HAS_CUDA)
- # Assuming building in a container that doesn't have CUDA installed (ie CPU-only build machine)
- # so use the stub cuda driver shared library
- if(EXISTS ${CUDA_LIBRARY_PATH}/stubs/libcuda.so)
- link_directories(${CUDA_LIBRARY_PATH}/stubs)
- endif()
- endif()
+ # In case of building on a production-like build container which may not have Cuda installed
+ if(NOT CMAKE_SYSTEM_HAS_CUDA)
+ # Assuming building in a container that doesn't have CUDA installed (ie CPU-only build machine)
+ # so use the stub cuda driver shared library
+ if(EXISTS ${CUDA_LIBRARY_PATH}/stubs/libcuda.so)
+ link_directories(${CUDA_LIBRARY_PATH}/stubs)
+ endif()
+ endif()
endif()
endif()
endif()
@@ -574,6 +592,9 @@ else()
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /EHsc")
endif()
+#
+# MXNet libraries
+#
set(MXNET_INSTALL_TARGETS mxnet)
if(UNIX)
list(APPEND MXNET_INSTALL_TARGETS mxnet_static)
@@ -586,12 +607,11 @@ else()
add_library(mxnet SHARED ${SOURCE})
endif()
-if(USE_CUDA)
- if(FIRST_CUDA AND MSVC)
- target_compile_options(mxnet PUBLIC "$<$<CONFIG:DEBUG>:-Xcompiler=-MTd>")
- target_compile_options(mxnet PUBLIC "$<$<CONFIG:RELEASE>:-Xcompiler=-MT>")
- endif()
+if(USE_CUDA AND FIRST_CUDA AND MSVC)
+ target_compile_options(mxnet PUBLIC "$<$<CONFIG:DEBUG>:-Xcompiler=-MTd>")
+ target_compile_options(mxnet PUBLIC "$<$<CONFIG:RELEASE>:-Xcompiler=-MT>")
endif()
+
if(USE_DIST_KVSTORE)
if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/ps-lite/CMakeLists.txt)
add_subdirectory("ps-lite")
@@ -621,6 +641,27 @@ if(USE_PLUGINS_WARPCTC)
target_link_libraries(mxnet PUBLIC optimized ${WARPCTC_LIB_RELEASE})
endif()
+#
+# BEGIN Cython Build
+#
+include(cmake/CythonUtil.cmake)
+mxnet_external_build_cython(2 target_2)
+mxnet_external_build_cython(3 target_3)
+
+if(INSTALL_CYTHON_INPLACE)
+ set(cython_install_targets mxnet ${target_2} ${target_3})
+ cython_install_into_source_dir(
+ ${CMAKE_CURRENT_BINARY_DIR}/python
+ ${CMAKE_CURRENT_SOURCE_DIR}/python
+ cython_install_targets
+ )
+endif()
+
+add_custom_target(mxnet_runtime DEPENDS ${cython_install_targets})
+#
+# END Cython build
+#
+
if(USE_OPENCV)
add_executable(im2rec "tools/im2rec.cc")
diff --git a/cmake/CythonUtil.cmake b/cmake/CythonUtil.cmake
new file mode 100644
index 0000000..f98bf61
--- /dev/null
+++ b/cmake/CythonUtil.cmake
@@ -0,0 +1,297 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+################################################################################################
+# Build a cython module
+#
+# Usage:
+# mxnet_external_build_cython(<python major version>)
+#
+function(mxnet_build_cython_module python_version _cython_modules)
+ string(REGEX REPLACE "@" ";" PROP_MXNET_INCLUDE_DIRECTORIES "${MXNET_INCLUDE_DIRECTORIES}")
+ string(REGEX REPLACE "@" ";" PROP_MXNET_INTERFACE_LINK_LIBRARIES "${MXNET_INTERFACE_LINK_LIBRARIES}")
+
+ foreach(var ${PROP_MXNET_INCLUDE_DIRECTORIES})
+ include_directories(${var})
+ endforeach()
+
+ unset(PYTHONLIBS_FOUND)
+ unset(PYTHON_LIBRARIES)
+ unset(PYTHON_INCLUDE_PATH)
+ unset(PYTHON_INCLUDE_DIRS)
+ unset(PYTHON_DEBUG_LIBRARIES)
+ unset(PYTHONLIBS_VERSION_STRING)
+ unset(PYTHONINTERP_FOUND)
+ unset(PYTHON_EXECUTABLE)
+ unset(PYTHON_VERSION_STRING)
+ unset(PYTHON_VERSION_MAJOR)
+ unset(PYTHON_VERSION_MINOR)
+ unset(PYTHON_VERSION_PATCH)
+
+ if(python_version EQUAL 2)
+ set(Python_ADDITIONAL_VERSIONS 2.7 2.6 2.5 2.4 2.3 2.2 2.1 2.0)
+ elseif(python_version EQUAL 3)
+ set(Python_ADDITIONAL_VERSIONS 3.7 3.6 3.5 3.4 3.3 3.2 3.1 3.0)
+ else()
+ message(FATAL_ERROR "Nov alid python_version set (must be 2 or 3)")
+ return()
+ endif()
+
+ set(python_libs_version ${python_version})
+ include(${MXNET_ROOT_DIR}/cmake/UseCython.cmake) # set from mxnet_external_build_cython
+
+ if(NOT CYTHON${python_version}_FOUND)
+ message(WARNING " Could not build cython target for Python ${python_version}")
+ return()
+ endif()
+
+ set(CYTHON_SUBDIR ".")
+
+ file(GLOB_RECURSE CYTHON_SOURCE "${MXNET_ROOT_DIR}/python/mxnet/*.pyx")
+
+ if(NOT MXNET_LIB_LOCATION)
+ set(MXNET_LIB_LOCATION mxnet)
+ endif()
+
+ set(CYTHON_CXX_SOURCE "")
+ foreach(cy_file ${CYTHON_SOURCE})
+ set_source_files_properties(${cy_file} PROPERTIES CYTHON_IS_CXX TRUE)
+ list(APPEND CYTHON_CXX_SOURCE ${cy_file_generated})
+ get_filename_component(cy_module ${cy_file} NAME_WE)
+ get_filename_component(cy_dir ${cy_file} DIRECTORY)
+# message(STATUS "MXNET_ROOT_DIR: ${MXNET_ROOT_DIR}")
+
+ file(RELATIVE_PATH cy_directory "${MXNET_ROOT_DIR}" ${cy_dir})
+# message(STATUS "cy_directory: ${cy_directory}")
+# message(STATUS "MXNET_BINARY_DIR: ${MXNET_BINARY_DIR}")
+ set(bin_directory "${MXNET_BINARY_DIR}/${cy_directory}")
+# message(STATUS "bin_directory: ${bin_directory}")
+ set(CYTHON_SUBDIR "${bin_directory}/cy${python_version}")
+ #message(STATUS "CYTHON_SUBDIR: ${CYTHON_SUBDIR}")
+ file(MAKE_DIRECTORY ${CYTHON_SUBDIR})
+
+ # We need cmake to have different target names for python 2 and 3
+ set(cy_module_name ${cy_module})
+
+ # cython_add_module expects cxx outyput dir to be relative to current binary dir
+ file(RELATIVE_PATH CYTHON_SUBDIR "${CMAKE_CURRENT_BINARY_DIR}" ${CYTHON_SUBDIR})
+ #${CMAKE_CURRENT_BINARY_DIR}/${c_cxx_output_subdir}
+
+ cython_add_module(${cy_module_name}
+ "${CYTHON_SUBDIR}"
+ "${MXNET_BINARY_DIR}/cython/cy${python_version}"
+ ${cy_file})
+
+ set_target_properties(${cy_module_name}
+ PROPERTIES
+ LIBRARY_OUTPUT_DIRECTORY "${CYTHON_SUBDIR}/"
+ INTERFACE_LINK_LIBRARIES "${PROP_MXNET_INTERFACE_LINK_LIBRARIES}"
+ )
+ target_link_libraries(${cy_module_name} ${MXNET_LIB_LOCATION})
+ list(APPEND output_cython_modules ${cy_module_name})
+ endforeach()
+ set(${_cython_modules} ${output_cython_modules} PARENT_SCOPE)
+ #message(STATUS "output_cython_modules: ${output_cython_modules}")
+endfunction()
+
+################################################################################################
+# Copy cython modules into source python dir
+#function(_cython_install_into_source_dir python_version _cython_binary_dir _source_root_dir)
+# file(GLOB_RECURSE cython_module_runtimes
+# "${_cython_binary_dir}/*.so"
+# "${_cython_binary_dir}/*.dll"
+# "${_cython_binary_dir}/*.pyd")
+# #message(STATUS "cython_module_runtimes: ${cython_module_runtimes}")
+# set(_running_target_suffix "")
+# foreach(_file ${cython_module_runtimes})
+# set(_running_target_suffix "${_running_target_suffix}_")
+# message(STATUS "_file: ${_file}")
+# get_filename_component(_cy_module_name ${_file} NAME_WE)
+# get_filename_component(_cy_module_directory ${_file} DIRECTORY)
+# #message(STATUS "_cy_module_name: ${_cy_module_name}")
+# #message(STATUS "_cy_module_directory: ${_cy_module_directory}")
+# file(RELATIVE_PATH _relpath_source ${_cython_binary_dir} ${_file})
+# #message(STATUS "_relpath_source: ${_relpath_source}")
+# set(_dest_file "${_source_root_dir}/${_relpath_source}")
+# #message(STATUS "_dest_file: ${_dest_file}")
+# get_filename_component(_dest_file_dir ${_dest_file} DIRECTORY)
+# file(MAKE_DIRECTORY ${_dest_file_dir})
+# #message(STATUS "${_file} -> ${_dest_file}")
+# set(_target_name cython_${_cy_module_name}_copy_shared_obj)
+# #message(STATUS "target: ${_target_name}")
+# add_custom_target(${_target_name} ALL
+# DEPENDS ${_cy_module_name}
+# COMMAND ${CMAKE_COMMAND} -E copy_if_different ${_file} ${_dest_file})
+# add_custom_target(cython_${_cy_module_name}_copy_cython_debug ALL
+# DEPENDS ${_cy_module_name}
+# COMMAND ${CMAKE_COMMAND} -E copy_directory
+# ${_cy_module_directory}/cython_debug
+# ${_dest_file_dir}/cython_debug
+# )
+# endforeach()
+#endfunction()
+
+function(cython_install_into_source_dir
+ _cython_binary_dir
+ _source_root_dir
+ _dependencies
+ )
+ file(GLOB_RECURSE cython_module_runtimes
+ "${_cython_binary_dir}/*.so"
+ "${_cython_binary_dir}/*.dll"
+ "${_cython_binary_dir}/*.pyd")
+ #message(STATUS "cython_module_runtimes: ${cython_module_runtimes}")
+ set(_running_target_suffix "")
+ foreach(_file ${cython_module_runtimes})
+ #set(_running_target_suffix "${_running_target_suffix}_")
+ #message(STATUS "_file: ${_file}")
+ get_filename_component(_cy_module_name ${_file} NAME_WE)
+ get_filename_component(_cy_module_directory ${_file} DIRECTORY)
+ #message(STATUS "_cy_module_name: ${_cy_module_name}")
+ #message(STATUS "_cy_module_directory: ${_cy_module_directory}")
+ file(RELATIVE_PATH _relpath_source ${_cython_binary_dir} ${_file})
+ #message(STATUS "_relpath_source: ${_relpath_source}")
+ set(_dest_file "${_source_root_dir}/${_relpath_source}")
+ #message(STATUS "_dest_file: ${_dest_file}")
+ get_filename_component(_dest_file_dir ${_dest_file} DIRECTORY)
+ file(MAKE_DIRECTORY ${_dest_file_dir})
+ #message(STATUS "${_file} -> ${_dest_file}")
+ #message(STATUS "target: ${_target_name}")
+ #message(STATUS "_dependencies: ${${_dependencies}}")
+ file(RELATIVE_PATH _full_module_name ${CMAKE_CURRENT_SOURCE_DIR} ${_dest_file})
+ string(REGEX REPLACE "/" "." _full_module_name "${_full_module_name}")
+ string(REGEX REPLACE "\\\\" "." _full_module_name "${_full_module_name}")
+ #message(STATUS "_full_module_name: ${_full_module_name}")
+if(UNIX)
+ add_custom_target(
+ ${_full_module_name} ALL
+ DEPENDS ${${_dependencies}}
+ COMMAND ln -sf ${_file} ${_dest_file})
+else()
+ add_custom_target(
+ ${_full_module_name} ALL
+ DEPENDS ${${_dependencies}}
+ COMMAND ${CMAKE_COMMAND} -E copy ${_file} ${_dest_file})
+endif()
+ endforeach()
+endfunction()
+
+################################################################################################
+# Spawn external CMakeLists.txt in order to build a particular cython/python version
+#
+# The spawn approach is because we need to detect and build with both python version 2 and 3
+# This is not osmething that a single process of cmake can deal with, so we launch
+# a cmake process for the cython build 2, then for 3, passing it our relevant config
+#
+# Usage:
+# mxnet_external_build_cython(<python major version>)
+#
+function(mxnet_external_build_cython python_major_version target)
+ set(PMV ${python_major_version})
+
+ if(CYTHON_WITHOUT_MXNET_TARGET)
+ set(CYTHON_DEPS "")
+ set(CYTHON_MXNET_LIB_LOCATION "")
+ else()
+ set(CYTHON_DEPS mxnet)
+ set(CYTHON_MXNET_LIB_LOCATION $<TARGET_LINKER_FILE:mxnet>)
+ endif()
+
+ file(GLOB_RECURSE CYTHON_SOURCE "python/mxnet/cython/*.pyx")
+
+ get_cmake_property(CACHE_VARS CACHE_VARIABLES)
+ foreach(_cache_var ${CACHE_VARS})
+ #message(STATUS "${_cache_var}=${${_cache_var}}")
+ get_property(CACHE_VAR_HELPSTRING CACHE ${_cache_var} PROPERTY HELPSTRING)
+ if(NOT _cache_var MATCHES "CMAKE_EXTRA_GENERATOR_.+"
+ AND NOT _cache_var MATCHES "FIND_PACKAGE_MESSAGE_.+"
+ )
+ if(_cache_var MATCHES "USE_.+"
+ OR _cache_var MATCHES "CMAKE_MODULE_.+"
+ OR CACHE_VAR_HELPSTRING STREQUAL "No help, variable specified on the command line."
+ )
+ get_property(_cache_var_type CACHE ${_cache_var} PROPERTY TYPE)
+ if(_cache_var_type STREQUAL "UNINITIALIZED")
+ set(_cache_var_type)
+ else()
+ set(_cache_var_type :${_cache_var_type})
+ endif()
+ set(CMAKE_ARGS "${CMAKE_ARGS} -D${_cache_var}${_cache_var_type}=\"${${_cache_var}}\"")
+ endif()
+ endif()
+ endforeach()
+
+ get_property(PROP_MXNET_INCLUDE_DIRECTORIES TARGET mxnet PROPERTY INCLUDE_DIRECTORIES)
+ string(REGEX REPLACE "\;" "@" MXNET_INCLUDE_DIRECTORIES "${PROP_MXNET_INCLUDE_DIRECTORIES}")
+
+ get_property(PROP_MXNET_INTERFACE_LINK_LIBRARIES TARGET mxnet PROPERTY INTERFACE_LINK_LIBRARIES)
+ string(REGEX REPLACE "\;" "@" MXNET_INTERFACE_LINK_LIBRARIES "${PROP_MXNET_INTERFACE_LINK_LIBRARIES}")
+
+ set(CYTHON_BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR}/cython/cy${PMV})
+
+ set(_config_cleanup_files "")
+ set(_config_cleanup_files_ex "")
+
+ foreach(_file ${CYTHON_SOURCE})
+ get_filename_component(_cy_module ${_file} NAME_WE)
+ list(APPEND _config_cleanup_files "${CYTHON_BINARY_DIR}/${_cy_module}.so")
+ list(APPEND _config_cleanup_files "${CYTHON_BINARY_DIR}/${_cy_module}.cxx")
+ list(APPEND _config_cleanup_files "${CYTHON_BINARY_DIR}/${_cy_module}.c")
+ endforeach()
+
+ # Clear some cmake-generated files
+ list(APPEND _config_cleanup_files_ex "${CYTHON_BINARY_DIR}/CMakeCache.txt")
+ list(APPEND _config_cleanup_files_ex "${CYTHON_BINARY_DIR}/Makefile")
+
+ # Get current cleanup files
+ get_directory_property(CLEANUP_FILES ADDITIONAL_MAKE_CLEAN_FILES)
+ list(APPEND CLEANUP_FILES ${_config_cleanup_files})
+ list(APPEND CLEANUP_FILES ${_config_cleanup_files_ex})
+ # Set new list of cleanup files
+ set_directory_properties(PROPERTIES ADDITIONAL_MAKE_CLEAN_FILES "${CLEANUP_FILES}")
+
+ file(MAKE_DIRECTORY ${CYTHON_BINARY_DIR})
+
+ add_custom_target(${PROJECT_NAME}_ConfigCython${PMV} ALL
+ ${CMAKE_COMMAND}
+ ${CMAKE_ARGS}
+ -G "${CMAKE_GENERATOR}"
+ -DCMAKE_MODULE_PATH="${CMAKE_MODULE_PATH}"
+ -DMXNET_INCLUDE_DIRECTORIES="${MXNET_INCLUDE_DIRECTORIES}"
+ -DMXNET_INTERFACE_LINK_LIBRARIES="${MXNET_INTERFACE_LINK_LIBRARIES}"
+ -DMXNET_LIB_LOCATION=${CYTHON_MXNET_LIB_LOCATION}
+ -DMXNET_ROOT_DIR=${CMAKE_CURRENT_SOURCE_DIR}
+ -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
+ -DCMAKE_C_FLAGS=${CMAKE_C_FLAGS}
+ -DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS}
+ #-DINSTALL_CYTHON_INPLACE=${INSTALL_CYTHON_INPLACE}
+ -DMXNET_BINARY_DIR=${CMAKE_CURRENT_BINARY_DIR}
+ ${CMAKE_CURRENT_SOURCE_DIR}/src/cython/cy${PMV}
+ WORKING_DIRECTORY ${CYTHON_BINARY_DIR}
+ DEPENDS ${CYTHON_DEPS}
+ )
+
+ add_custom_target(${PROJECT_NAME}_BuildCython${PMV} ALL
+ ${CMAKE_COMMAND}
+ --build ${CYTHON_BINARY_DIR}
+ --config ${CMAKE_BUILD_TYPE}
+ WORKING_DIRECTORY ${CYTHON_BINARY_DIR}
+ DEPENDS ${PROJECT_NAME}_ConfigCython${PMV}
+ )
+
+ set(${target} ${PROJECT_NAME}_BuildCython${PMV} PARENT_SCOPE)
+endfunction()
diff --git a/cmake/Modules/FindCython.cmake b/cmake/Modules/FindCython.cmake
new file mode 100644
index 0000000..132de50
--- /dev/null
+++ b/cmake/Modules/FindCython.cmake
@@ -0,0 +1,69 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Find the Cython compiler.
+#
+# This code sets the following variables:
+#
+# CYTHON_EXECUTABLE
+#
+# See also UseCython.cmake
+
+#=============================================================================
+# Copyright 2011 Kitware, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#=============================================================================
+
+# Use the Cython executable that lives next to the Python executable
+# if it is a local installation.
+
+if(PACKAGE_FIND_VERSION_MAJOR EQUAL 3)
+ set(CYTHON_EXE_NAMES cython3 cython.bat cython)
+ message(STATUS " Looking for Cython version 3")
+else()
+ set(CYTHON_EXE_NAMES cython.bat cython cython3)
+endif()
+
+if(PYTHONINTERP_FOUND)
+ get_filename_component( _python_path ${PYTHON_EXECUTABLE} PATH )
+ find_program(CYTHON_EXECUTABLE
+ NAMES ${CYTHON_EXE_NAMES}
+ HINTS ${_python_path}
+ )
+else()
+ find_program(CYTHON_EXECUTABLE NAMES ${CYTHON_EXE_NAMES})
+endif()
+
+include( FindPackageHandleStandardArgs )
+find_package_handle_standard_args(Cython DEFAULT_MSG CYTHON_EXECUTABLE)
+
+if(CYTHON_FOUND)
+ message(STATUS "Found Cython (executable: ${CYTHON_EXECUTABLE})")
+ mark_as_advanced( CYTHON_EXECUTABLE )
+endif()
+
diff --git a/cmake/Modules/FindOpenBLAS.cmake b/cmake/Modules/FindOpenBLAS.cmake
index a3a79ca..e325b9e 100644
--- a/cmake/Modules/FindOpenBLAS.cmake
+++ b/cmake/Modules/FindOpenBLAS.cmake
@@ -15,7 +15,6 @@
# specific language governing permissions and limitations
# under the License.
-file(TO_CMAKE_PATH "$ENV{OpenBLAS_HOME}" OpenBLAS_HOME)
file(TO_CMAKE_PATH "$ENV{OpenBLAS}" OpenBLAS_DIR)
SET(Open_BLAS_INCLUDE_SEARCH_PATHS
@@ -31,6 +30,8 @@ SET(Open_BLAS_INCLUDE_SEARCH_PATHS
${PROJECT_SOURCE_DIR}/thirdparty/OpenBLAS/include
${OpenBLAS_HOME}
${OpenBLAS_HOME}/include
+ $ENV{OpenBLAS_HOME}
+ $ENV{OpenBLAS_HOME}/include
)
SET(Open_BLAS_LIB_SEARCH_PATHS
@@ -46,10 +47,12 @@ SET(Open_BLAS_LIB_SEARCH_PATHS
/usr/local/opt/openblas/lib
${PROJECT_SOURCE_DIR}/3rdparty/OpenBLAS/lib
${PROJECT_SOURCE_DIR}/thirdparty/OpenBLAS/lib
- ${OpenBLAS_DIR}
- ${OpenBLAS_DIR}/lib
+ ${OpenBLAS_DIR}
+ ${OpenBLAS_DIR}/lib
${OpenBLAS_HOME}
${OpenBLAS_HOME}/lib
+ $ENV{OpenBLAS_HOME}
+ $ENV{OpenBLAS_HOME}/lib
)
FIND_PATH(OpenBLAS_INCLUDE_DIR NAMES cblas.h PATHS ${Open_BLAS_INCLUDE_SEARCH_PATHS})
diff --git a/cmake/UseCython.cmake b/cmake/UseCython.cmake
new file mode 100644
index 0000000..26969dd
--- /dev/null
+++ b/cmake/UseCython.cmake
@@ -0,0 +1,390 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Define a function to create Cython modules.
+#
+# For more information on the Cython project, see http://cython.org/.
+# "Cython is a language that makes writing C extensions for the Python language
+# as easy as Python itself."
+#
+# This file defines a CMake function to build a Cython Python module.
+# To use it, first include this file.
+#
+# include( UseCython )
+#
+# Then call cython_add_module to create a module.
+#
+# cython_add_module( <module_name> <src1> <src2> ... <srcN> )
+#
+# To create a standalone executable, the function
+#
+# cython_add_standalone_executable( <executable_name> [MAIN_MODULE src1] <src1> <src2> ... <srcN> )
+#
+# To avoid dependence on Python, set the PYTHON_LIBRARY cache variable to point
+# to a static library. If a MAIN_MODULE source is specified,
+# the "if __name__ == '__main__':" from that module is used as the C main() method
+# for the executable. If MAIN_MODULE, the source with the same basename as
+# <executable_name> is assumed to be the MAIN_MODULE.
+#
+# Where <module_name> is the name of the resulting Python module and
+# <src1> <src2> ... are source files to be compiled into the module, e.g. *.pyx,
+# *.py, *.c, *.cxx, etc. A CMake target is created with name <module_name>. This can
+# be used for target_link_libraries(), etc.
+#
+# The sample paths set with the CMake include_directories() command will be used
+# for include directories to search for *.pxd when running the Cython complire.
+#
+# Cache variables that effect the behavior include:
+#
+# CYTHON_ANNOTATE
+# CYTHON_NO_DOCSTRINGS
+# CYTHON_FLAGS
+#
+# Source file properties that effect the build process are
+#
+# CYTHON_IS_CXX
+#
+# If this is set of a *.pyx file with CMake set_source_files_properties()
+# command, the file will be compiled as a C++ file.
+#
+# See also FindCython.cmake
+
+#=============================================================================
+# Copyright 2011 Kitware, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#=============================================================================
+
+# Configuration options.
+set( CYTHON_ANNOTATE OFF
+ CACHE BOOL "Create an annotated .html file when compiling *.pyx." )
+set( CYTHON_NO_DOCSTRINGS OFF
+ CACHE BOOL "Strip docstrings from the compiled module." )
+set( CYTHON_FLAGS "" CACHE STRING
+ "Extra flags to the cython compiler." )
+mark_as_advanced( CYTHON_ANNOTATE CYTHON_NO_DOCSTRINGS CYTHON_FLAGS )
+
+unset(PYTHONLIBS_FOUND)
+unset(PYTHONINTERP_FOUND)
+unset(CYTHON_FOUND)
+
+if(NOT python_libs_version)
+ message(STATUS "Looking for python dependencies, version: ${python_libs_version}")
+endif()
+
+find_package(PythonInterp ${python_libs_version} REQUIRED)
+if(PYTHONINTERP_FOUND)
+ message(STATUS "Python ${python_libs_version} executable: ${PYTHON_EXECUTABLE}")
+ find_package(PythonLibs ${python_libs_version} REQUIRED)
+ if(PYTHONLIBS_FOUND)
+ set(PYTHON_DEBUG_LIBRARY ${PYTHON_LIBRARY})
+ set(PYTHON_DEBUG_LIBRARIES ${PYTHON_DEBUG_LIBRARIES})
+ find_package(Cython ${python_libs_version} REQUIRED)
+ if(CYTHON_FOUND)
+ set(CYTHON${python_libs_version}_FOUND ${python_libs_version})
+ message(STATUS " CYTHON${python_libs_version}_FOUND: ${CYTHON${python_libs_version}_FOUND}")
+ else()
+ message(WARNING " Could not find package: Cython")
+ endif()
+ else()
+ message(WARNING " Could not find package: PythonLibs")
+ endif()
+else()
+ message(WARNING " Could not find package: PythonInterp")
+endif()
+
+if(NOT CYTHON${python_libs_version}_FOUND)
+ message(WARNING " Could not find cython and/or dependencies for major version ${python_libs_version}")
+ return()
+endif()
+
+message(STATUS "PYTHONLIBS_VERSION_STRING: ${PYTHONLIBS_VERSION_STRING}")
+string(REPLACE "." ";" PYTHON_VERSION_LIST ${PYTHONLIBS_VERSION_STRING})
+list(GET PYTHON_VERSION_LIST 0 PYTHON_VERSION_MAJOR)
+list(GET PYTHON_VERSION_LIST 1 PYTHON_VERSION_MINOR)
+list(GET PYTHON_VERSION_LIST 1 PYTHON_VERSION_PATCH)
+
+if(NOT PYTHON_VERSION_MAJOR EQUAL ${python_libs_version})
+ message(WARNING "Scripts found wrong python major version: ${PYTHON_VERSION_MAJOR} instead of ${python_libs_version}. This is most likely due to version ${python_libs_version} not being installed or not found")
+ unset(PYTHONLIBS_FOUND)
+ unset(PYTHONINTERP_FOUND)
+ unset(CYTHON_FOUND)
+ return()
+endif()
+
+set( CYTHON_CXX_EXTENSION "cxx" )
+set( CYTHON_C_EXTENSION "c" )
+
+# Create a *.c or *.cxx file from a *.pyx file.
+# Input the generated file basename. The generate file will put into the variable
+# placed in the "generated_file" argument. Finally all the *.py and *.pyx files.
+function( compile_pyx _name c_cxx_output_subdir debug_output_dir generated_file )
+ # Default to assuming all files are C.
+ set( cxx_arg "" )
+ set( extension ${CYTHON_C_EXTENSION} )
+ set( pyx_lang "C" )
+ set( comment "Compiling Cython C source for ${_name}..." )
+
+ set( cython_include_directories "" )
+ set( pxd_dependencies "" )
+ set( pxi_dependencies "" )
+ set( c_header_dependencies "" )
+ set( pyx_locations "" )
+
+ #message(STATUS " compile_pyx( ${_name} ${generated_file} ${ARGN} )")
+
+ foreach( pyx_file ${ARGN} )
+ get_filename_component( pyx_file_basename "${pyx_file}" NAME_WE )
+
+ # Determine if it is a C or C++ file.
+ get_source_file_property( property_is_cxx ${pyx_file} CYTHON_IS_CXX )
+ if( ${property_is_cxx} )
+ set( cxx_arg "--cplus" )
+ set( extension ${CYTHON_CXX_EXTENSION} )
+ set( pyx_lang "CXX" )
+ set( comment "Compiling Cython CXX source for ${_name}..." )
+ endif()
+
+ # Get the include directories.
+ get_source_file_property( pyx_location ${pyx_file} LOCATION )
+
+ get_filename_component( pyx_path ${pyx_location} PATH )
+
+ #get_directory_property( cmake_include_directories DIRECTORY ${pyx_path} INCLUDE_DIRECTORIES )
+
+ list( APPEND cython_include_directories ${cmake_include_directories} )
+ list( APPEND pyx_locations "${pyx_location}" )
+
+ # Determine dependencies.
+ # Add the pxd file will the same name as the given pyx file.
+ unset( corresponding_pxd_file CACHE )
+ find_file( corresponding_pxd_file ${pyx_file_basename}.pxd
+ PATHS "${pyx_path}" ${cmake_include_directories}
+ NO_DEFAULT_PATH )
+ if( corresponding_pxd_file )
+ list( APPEND pxd_dependencies "${corresponding_pxd_file}" )
+ endif()
+
+ # Look for included pxi files
+ file(STRINGS "${pyx_file}" include_statements REGEX "include +['\"]([^'\"]+).*")
+ foreach(statement ${include_statements})
+ string(REGEX REPLACE "include +['\"]([^'\"]+).*" "\\1" pxi_file "${statement}")
+ unset(pxi_location CACHE)
+ find_file(pxi_location ${pxi_file}
+ PATHS "${pyx_path}" ${cmake_include_directories} NO_DEFAULT_PATH)
+ if (pxi_location)
+ list(APPEND pxi_dependencies ${pxi_location})
+ get_filename_component( found_pyi_file_basename "${pxi_file}" NAME_WE )
+ get_filename_component( found_pyi_path ${pxi_location} PATH )
+ unset( found_pyi_pxd_file CACHE )
+ find_file( found_pyi_pxd_file ${found_pyi_file_basename}.pxd
+ PATHS "${found_pyi_path}" ${cmake_include_directories} NO_DEFAULT_PATH )
+ if (found_pyi_pxd_file)
+ list( APPEND pxd_dependencies "${found_pyi_pxd_file}" )
+ endif()
+ endif()
+ endforeach() # for each include statement found
+
+ # pxd files to check for additional dependencies.
+ set( pxds_to_check "${pyx_file}" "${pxd_dependencies}" )
+ set( pxds_checked "" )
+ set( number_pxds_to_check 1 )
+ while( ${number_pxds_to_check} GREATER 0 )
+ foreach( pxd ${pxds_to_check} )
+ list( APPEND pxds_checked "${pxd}" )
+ list( REMOVE_ITEM pxds_to_check "${pxd}" )
+
+ # check for C header dependencies
+ file( STRINGS "${pxd}" extern_from_statements
+ REGEX "cdef[ ]+extern[ ]+from.*$" )
+ foreach( statement ${extern_from_statements} )
+ # Had trouble getting the quote in the regex
+ string( REGEX REPLACE "cdef[ ]+extern[ ]+from[ ]+[\"]([^\"]+)[\"].*" "\\1" header "${statement}" )
+ unset( header_location CACHE )
+ find_file( header_location ${header} PATHS ${cmake_include_directories} )
+ if( header_location )
+ list( FIND c_header_dependencies "${header_location}" header_idx )
+ if( ${header_idx} LESS 0 )
+ list( APPEND c_header_dependencies "${header_location}" )
+ endif()
+ endif()
+ endforeach()
+
+ # check for pxd dependencies
+
+ # Look for cimport statements.
+ set( module_dependencies "" )
+ file( STRINGS "${pxd}" cimport_statements REGEX cimport )
+ foreach( statement ${cimport_statements} )
+ if( ${statement} MATCHES from )
+ string( REGEX REPLACE "from[ ]+([^ ]+).*" "\\1" module "${statement}" )
+ else()
+ string( REGEX REPLACE "cimport[ ]+([^ ]+).*" "\\1" module "${statement}" )
+ endif()
+ list( APPEND module_dependencies ${module} )
+ endforeach()
+ list( REMOVE_DUPLICATES module_dependencies )
+ # Add the module to the files to check, if appropriate.
+ foreach( module ${module_dependencies} )
+ unset( pxd_location CACHE )
+ find_file( pxd_location ${module}.pxd
+ PATHS "${pyx_path}" ${cmake_include_directories} NO_DEFAULT_PATH )
+ if( pxd_location )
+ list( FIND pxds_checked ${pxd_location} pxd_idx )
+ if( ${pxd_idx} LESS 0 )
+ list( FIND pxds_to_check ${pxd_location} pxd_idx )
+ if( ${pxd_idx} LESS 0 )
+ list( APPEND pxds_to_check ${pxd_location} )
+ list( APPEND pxd_dependencies ${pxd_location} )
+ endif() # if it is not already going to be checked
+ endif() # if it has not already been checked
+ endif() # if pxd file can be found
+ endforeach() # for each module dependency discovered
+ endforeach() # for each pxd file to check
+ list( LENGTH pxds_to_check number_pxds_to_check )
+ endwhile()
+
+
+
+ endforeach() # pyx_file
+
+ # Set additional flags.
+ if( CYTHON_ANNOTATE )
+ set( annotate_arg "--annotate" )
+ endif()
+
+ if( CYTHON_NO_DOCSTRINGS )
+ set( no_docstrings_arg "--no-docstrings" )
+ endif()
+
+ if( "${CMAKE_BUILD_TYPE}" STREQUAL "Debug" OR
+ "${CMAKE_BUILD_TYPE}" STREQUAL "RelWithDebInfo" )
+ set( cython_debug_arg
+ #"--gdb"
+ --gdb-outdir ${debug_output_dir}
+ )
+ endif()
+
+ if( "${PYTHONLIBS_VERSION_STRING}" MATCHES "^2." )
+ set( version_arg "-2" )
+ elseif( "${PYTHONLIBS_VERSION_STRING}" MATCHES "^3." )
+ set( version_arg "-3" )
+ else()
+ set( version_arg )
+ endif()
+
+ # Include directory arguments.
+ list( REMOVE_DUPLICATES cython_include_directories )
+ set( include_directory_arg "" )
+ foreach( _include_dir ${cython_include_directories} )
+ set( include_directory_arg ${include_directory_arg} "-I" "${_include_dir}" )
+ endforeach()
+
+ # Determining generated file name.
+ set( _generated_file "${CMAKE_CURRENT_BINARY_DIR}/${c_cxx_output_subdir}/${_name}.${extension}" )
+ set_source_files_properties( ${_generated_file} PROPERTIES GENERATED TRUE )
+ set( ${generated_file} ${_generated_file} PARENT_SCOPE )
+
+ list( REMOVE_DUPLICATES pxd_dependencies )
+ list( REMOVE_DUPLICATES c_header_dependencies )
+
+ # Add the command to run the compiler.
+ add_custom_command( OUTPUT ${_generated_file}
+ COMMAND ${CYTHON_EXECUTABLE}
+ ARGS ${cxx_arg} ${include_directory_arg} ${version_arg}
+ ${annotate_arg} ${no_docstrings_arg} ${cython_debug_arg} ${CYTHON_FLAGS}
+ --output-file ${_generated_file} ${pyx_locations}
+ DEPENDS ${pyx_locations} ${pxd_dependencies} ${pxi_dependencies}
+ IMPLICIT_DEPENDS ${pyx_lang} ${c_header_dependencies}
+ WORKING_DIRECTORY ${c_cxx_output_subdir}
+ COMMENT ${comment}
+ )
+
+ # Remove their visibility to the user.
+ set( corresponding_pxd_file "" CACHE INTERNAL "" )
+ set( header_location "" CACHE INTERNAL "" )
+ set( pxd_location "" CACHE INTERNAL "" )
+endfunction()
+
+# cython_add_module( <name> src1 src2 ... srcN )
+# Build the Cython Python module.
+function( cython_add_module _name c_cxx_output_subdir debug_output_dir)
+ set( pyx_module_sources "" )
+ set( other_module_sources "" )
+ foreach( _file ${ARGN} )
+ if( ${_file} MATCHES ".*\\.py[x]?$" )
+ list( APPEND pyx_module_sources ${_file} )
+ else()
+ list( APPEND other_module_sources ${_file} )
+ endif()
+ endforeach()
+ compile_pyx( ${_name} ${c_cxx_output_subdir} ${debug_output_dir} generated_file ${pyx_module_sources})
+ include_directories( ${PYTHON_INCLUDE_DIRS} )
+ python_add_module( ${_name} ${generated_file} ${other_module_sources} )
+ if( APPLE )
+ set_target_properties( ${_name} PROPERTIES LINK_FLAGS "-undefined dynamic_lookup" )
+ else()
+ target_link_libraries( ${_name} ${PYTHON_LIBRARIES} )
+ endif()
+endfunction()
+
+include( CMakeParseArguments )
+# cython_add_standalone_executable( _name [MAIN_MODULE src3.py] src1 src2 ... srcN )
+# Creates a standalone executable the given sources.
+function( cython_add_standalone_executable _name debug_output_dir)
+ set( pyx_module_sources "" )
+ set( other_module_sources "" )
+ set( main_module "" )
+ cmake_parse_arguments( cython_arguments "" "MAIN_MODULE" "" ${ARGN} )
+ include_directories( ${PYTHON_INCLUDE_DIRS} )
+ foreach( _file ${cython_arguments_UNPARSED_ARGUMENTS} )
+ if( ${_file} MATCHES ".*\\.py[x]?$" )
+ get_filename_component( _file_we ${_file} NAME_WE )
+ if( "${_file_we}" STREQUAL "${_name}" )
+ set( main_module "${_file}" )
+ elseif( NOT "${_file}" STREQUAL "${cython_arguments_MAIN_MODULE}" )
+ set( PYTHON_MODULE_${_file_we}_static_BUILD_SHARED OFF )
+ compile_pyx( "${_file_we}_static" generated_file ${debug_output_dir} "${_file}" )
+ list( APPEND pyx_module_sources "${generated_file}" )
+ endif()
+ else()
+ list( APPEND other_module_sources ${_file} )
+ endif()
+ endforeach()
+
+ if( cython_arguments_MAIN_MODULE )
+ set( main_module ${cython_arguments_MAIN_MODULE} )
+ endif()
+ if( NOT main_module )
+ message( FATAL_ERROR "main module not found." )
+ endif()
+ get_filename_component( main_module_we "${main_module}" NAME_WE )
+ set( CYTHON_FLAGS ${CYTHON_FLAGS} --embed )
+ compile_pyx( "${main_module_we}_static" generated_file ${main_module} )
+ add_executable( ${_name} ${generated_file} ${pyx_module_sources} ${other_module_sources} )
+ target_link_libraries( ${_name} ${PYTHON_LIBRARIES} ${pyx_module_libs} )
+endfunction()
diff --git a/python/mxnet/_cy3/__init__.py b/python/mxnet/cython/__init__.py
similarity index 93%
copy from python/mxnet/_cy3/__init__.py
copy to python/mxnet/cython/__init__.py
index 44dcca5..13a8339 100644
--- a/python/mxnet/_cy3/__init__.py
+++ b/python/mxnet/cython/__init__.py
@@ -14,5 +14,3 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
-"""Namespace for cython generated modules for python3"""
diff --git a/python/mxnet/cython/base.pyi b/python/mxnet/cython/base.pyi
index d73e1a7..3963b37 100644
--- a/python/mxnet/cython/base.pyi
+++ b/python/mxnet/cython/base.pyi
@@ -1,4 +1,4 @@
-from ..base import MXNetError
+from ...base import MXNetError
from libcpp.vector cimport vector
from libcpp.string cimport string
diff --git a/python/mxnet/_cy3/README b/python/mxnet/cython/cy2/README
similarity index 100%
rename from python/mxnet/_cy3/README
rename to python/mxnet/cython/cy2/README
diff --git a/python/mxnet/_cy2/__init__.py b/python/mxnet/cython/cy2/__init__.py
similarity index 100%
copy from python/mxnet/_cy2/__init__.py
copy to python/mxnet/cython/cy2/__init__.py
diff --git a/python/mxnet/_cy2/README b/python/mxnet/cython/cy3/README
similarity index 100%
rename from python/mxnet/_cy2/README
rename to python/mxnet/cython/cy3/README
diff --git a/python/mxnet/_cy3/__init__.py b/python/mxnet/cython/cy3/__init__.py
similarity index 100%
copy from python/mxnet/_cy3/__init__.py
copy to python/mxnet/cython/cy3/__init__.py
diff --git a/python/mxnet/cython/mxcython.pyx b/python/mxnet/cython/mxcython.pyx
new file mode 100644
index 0000000..96a0034
--- /dev/null
+++ b/python/mxnet/cython/mxcython.pyx
@@ -0,0 +1,86 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from __future__ import absolute_import as _abs
+
+import sys as _sys
+import ctypes as _ctypes
+import numpy as np
+import time
+from ...ndarray_doc import _build_doc
+from libc.stdint cimport uint32_t, int64_t
+
+include "./base.pyi"
+
+# C API functions
+cdef extern from "../../../src/cython/cython_util.h":
+ int CythonPrintFromCPP(const char *foo);
+ int Printf(const char *fmt, ...);
+ int TrivialCPPCall(int var);
+ unsigned long long TimeInMilliseconds();
+
+# C++ Rectangle class
+cdef extern from "../../../src/cython/cython_util.h" namespace "shapes":
+ cdef cppclass Rectangle:
+ Rectangle() except +
+ Rectangle(int, int, int, int) except +
+ int x0, y0, x1, y1
+ int getArea()
+ void getSize(int* width, int* height)
+ void move(int, int)
+
+
+# Cython class: CythonTestClass
+cdef class CythonTestClass:
+ """Symbol is symbolic graph."""
+ # handle for symbolic operator.
+ cdef int cwritable
+
+ def __init__(self):
+ self.cwritable = 99
+
+ def print_something(self, str the_string):
+ print('BEFORE CythonPrintFromCPP')
+ CALL(CythonPrintFromCPP("This is from C++"))
+ print('AFTER CythonPrintFromCPP')
+ print('CythonTestClass::print_something( {} )'.format(the_string))
+
+def test_cpp_class():
+ cdef int recArea
+ rec_ptr = new Rectangle(1, 2, 3, 4)
+ try:
+ recArea = rec_ptr.getArea()
+ Printf("Printf() call: Area: %d\n", recArea)
+ finally:
+ del rec_ptr # delete heap allocated object
+
+def test_perf(int count, int make_c_call):
+ cdef unsigned long long start = TimeInMilliseconds()
+ cdef int foo = 0
+ cdef int i = 0
+ while i < count:
+ foo += i
+ if foo > count:
+ foo = 0
+ if make_c_call != 0:
+ TrivialCPPCall(0)
+ i += 1
+ cdef unsigned long long stop = TimeInMilliseconds()
+ Printf("CYTHON: %d items took %f seconds\n", count, float(stop - start)/1000)
+
+def print_pi(terms):
+ print(float(0.0))
diff --git a/python/mxnet/_cy2/__init__.py b/python/mxnet/ndarray/cy2/__init__.py
similarity index 100%
copy from python/mxnet/_cy2/__init__.py
copy to python/mxnet/ndarray/cy2/__init__.py
diff --git a/python/mxnet/_cy3/__init__.py b/python/mxnet/ndarray/cy3/__init__.py
similarity index 100%
copy from python/mxnet/_cy3/__init__.py
copy to python/mxnet/ndarray/cy3/__init__.py
diff --git a/python/mxnet/cython/ndarray.pyx b/python/mxnet/ndarray/ndarray.pyx
similarity index 98%
rename from python/mxnet/cython/ndarray.pyx
rename to python/mxnet/ndarray/ndarray.pyx
index 319dc49..57e207d 100644
--- a/python/mxnet/cython/ndarray.pyx
+++ b/python/mxnet/ndarray/ndarray.pyx
@@ -20,10 +20,10 @@ from __future__ import absolute_import as _abs
import sys as _sys
import ctypes as _ctypes
import numpy as np
-from ..ndarray_doc import _build_doc
+from ...ndarray_doc import _build_doc
from libc.stdint cimport uint32_t, int64_t
-include "./base.pyi"
+include "../cython/base.pyi"
cdef class NDArrayBase:
"""Symbol is symbolic graph."""
diff --git a/python/mxnet/_cy2/__init__.py b/python/mxnet/symbol/cy2/__init__.py
similarity index 100%
rename from python/mxnet/_cy2/__init__.py
rename to python/mxnet/symbol/cy2/__init__.py
diff --git a/python/mxnet/_cy3/__init__.py b/python/mxnet/symbol/cy3/__init__.py
similarity index 100%
copy from python/mxnet/_cy3/__init__.py
copy to python/mxnet/symbol/cy3/__init__.py
diff --git a/python/mxnet/cython/symbol.pyx b/python/mxnet/symbol/symbol.pyx
similarity index 96%
rename from python/mxnet/cython/symbol.pyx
rename to python/mxnet/symbol/symbol.pyx
index 1bdea6c..c3062d4 100644
--- a/python/mxnet/cython/symbol.pyx
+++ b/python/mxnet/symbol/symbol.pyx
@@ -22,11 +22,11 @@ import ctypes as _ctypes
import numpy as _numpy
from numbers import Number as _Number
-from ..name import NameManager
-from ..attribute import AttrScope
-from ..symbol_doc import _build_doc
+from ...name import NameManager
+from ...attribute import AttrScope
+from ...symbol_doc import _build_doc
-include "./base.pyi"
+include "../cython/base.pyi"
cdef class SymbolBase:
"""Symbol is symbolic graph."""
diff --git a/python/setup.py b/python/setup.py
index cf94adf..320001f 100644
--- a/python/setup.py
+++ b/python/setup.py
@@ -31,10 +31,13 @@ else:
kwargs = {'install_requires': ['numpy<=1.13.3,>=1.8.2', 'requests==2.18.4', 'graphviz==0.8.1'], 'zip_safe': False}
from setuptools import find_packages
-with_cython = False
+with_cython = True
if '--with-cython' in sys.argv:
with_cython = True
sys.argv.remove('--with-cython')
+if '--without-cython' in sys.argv:
+ with_cython = False
+ sys.argv.remove('--without-cython')
# We can not import `mxnet.info.py` in setup.py directly since mxnet/__init__.py
# Will be invoked which introduces dependences
diff --git a/python/mxnet/_cy3/__init__.py b/src/cython/cy2/CMakeLists.txt
similarity index 61%
copy from python/mxnet/_cy3/__init__.py
copy to src/cython/cy2/CMakeLists.txt
index 44dcca5..da67ba6 100644
--- a/python/mxnet/_cy3/__init__.py
+++ b/src/cython/cy2/CMakeLists.txt
@@ -15,4 +15,19 @@
# specific language governing permissions and limitations
# under the License.
-"""Namespace for cython generated modules for python3"""
+cmake_minimum_required(VERSION 3.0.2)
+
+project(cy2 C CXX)
+
+include(${CMAKE_CURRENT_SOURCE_DIR}/../../../cmake/CythonUtil.cmake)
+
+set(cython_modules "")
+
+mxnet_build_cython_module(2 cython_modules)
+#message(STATUS "cython_modules: ${cython_modules}")
+#if(INSTALL_CYTHON_INPLACE)
+# message(STATUS "Installing cython in-place")
+# cython_install_into_source_dir("${MXNET_BINARY_DIR}/python" "${MXNET_ROOT_DIR}/python")
+#else()
+# message(STATUS "Not installing cython in-place")
+#endif()
diff --git a/python/mxnet/_cy3/__init__.py b/src/cython/cy3/CMakeLists.txt
similarity index 61%
copy from python/mxnet/_cy3/__init__.py
copy to src/cython/cy3/CMakeLists.txt
index 44dcca5..e5df735 100644
--- a/python/mxnet/_cy3/__init__.py
+++ b/src/cython/cy3/CMakeLists.txt
@@ -15,4 +15,19 @@
# specific language governing permissions and limitations
# under the License.
-"""Namespace for cython generated modules for python3"""
+cmake_minimum_required(VERSION 3.0.2)
+
+project(cy3 C CXX)
+
+include(${CMAKE_CURRENT_SOURCE_DIR}/../../../cmake/CythonUtil.cmake)
+
+set(cython_modules "")
+
+mxnet_build_cython_module(3 cython_modules)
+#message(STATUS "cython_modules: ${cython_modules}")
+#if(INSTALL_CYTHON_INPLACE)
+# message(STATUS "Installing cython in-place")
+# cython_install_into_source_dir("${MXNET_BINARY_DIR}/python" "${MXNET_ROOT_DIR}/python")
+#else()
+# message(STATUS "Not installing cython in-place")
+#endif()
diff --git a/src/cython/cython_util.cc b/src/cython/cython_util.cc
new file mode 100644
index 0000000..7aeddbd
--- /dev/null
+++ b/src/cython/cython_util.cc
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+#include <iostream>
+#include <cstdarg>
+#include <sys/time.h>
+#include <chrono>
+#include "./cython_util.h"
+
+extern "C" int CythonPrintFromCPP(const char *foo) {
+ if(foo) {
+ std::cout << foo << std::endl << std::flush;
+ }
+ return 0;
+}
+
+extern "C" int Printf(const char *fmt, ...) {
+ va_list args;
+ va_start(args, fmt);
+ const int res = vprintf(fmt, args);
+ va_end(args);
+ return res;
+}
+
+extern "C" int TrivialCPPCall(int var) {
+ static int static_var = 0;
+ static_var = var;
+ return static_var;
+}
+
+using Tick = std::chrono::high_resolution_clock::time_point;
+static inline Tick Now() { return std::chrono::high_resolution_clock::now(); }
+
+static const Tick _app_start_time = Now();
+
+static inline uint64_t GetDurationInNanoseconds(const Tick &t1, const Tick &t2) {
+ return static_cast<uint64_t>(
+ std::chrono::duration_cast<std::chrono::nanoseconds>(t2 - t1).count());
+}
+
+static inline uint64_t GetDurationInNanoseconds(const Tick &since) {
+ return GetDurationInNanoseconds(since, Now());
+}
+
+constexpr size_t SLEEP_DURATION = 500;
+constexpr size_t TIMER_PERIOD = 10; // Ideal is 50 periods occur
+constexpr size_t MIN_COUNT_WHILE_SLEEPING = 10;
+constexpr size_t MAX_COUNT_WHILE_SLEEPING = 150;
+
+static inline size_t GetDurationInMilliseconds(const Tick& start_time) {
+ return static_cast<size_t>(GetDurationInNanoseconds(start_time)/1000/1000);
+}
+
+
+extern "C" uint64_t TimeInMilliseconds() {
+ return GetDurationInMilliseconds(_app_start_time);
+}
+
+namespace shapes {
+
+Rectangle::Rectangle() { }
+
+Rectangle::Rectangle(int X0, int Y0, int X1, int Y1) {
+ x0 = X0;
+ y0 = Y0;
+ x1 = X1;
+ y1 = Y1;
+}
+
+Rectangle::~Rectangle() { }
+
+int Rectangle::getArea() {
+ return (x1 - x0) * (y1 - y0);
+}
+
+void Rectangle::getSize(int *width, int *height) {
+ (*width) = x1 - x0;
+ (*height) = y1 - y0;
+}
+
+void Rectangle::move(int dx, int dy) {
+ x0 += dx;
+ y0 += dy;
+ x1 += dx;
+ y1 += dy;
+}
+
+} // namespace shapes
diff --git a/src/cython/cython_util.h b/src/cython/cython_util.h
new file mode 100644
index 0000000..8d519cf
--- /dev/null
+++ b/src/cython/cython_util.h
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+#ifndef MXNET_CYTHON_CYTHON_UTIL_H_
+#define MXNET_CYTHON_CYTHON_UTIL_H_
+
+/*! \brief Inhibit C++ name-mangling for MXNet functions. */
+#ifdef __cplusplus
+extern "C" {
+#endif // __cplusplus
+
+int CythonPrintFromCPP(const char *foo);
+int Printf(const char *fmt, ...);
+int TrivialCPPCall(int var);
+uint64_t TimeInMilliseconds();
+
+#ifdef __cplusplus
+}
+#endif // __cplusplus
+
+namespace shapes {
+
+class Rectangle {
+ public:
+ int x0, y0, x1, y1;
+ Rectangle();
+ Rectangle(int x0, int y0, int x1, int y1);
+ ~Rectangle();
+ int getArea();
+ void getSize(int* width, int* height);
+ void move(int dx, int dy);
+};
+
+} // namespace shapes
+
+#endif // MXNET_CYTHON_CYTHON_UTIL_H_
diff --git a/tests/python/unittest/test_cython.py b/tests/python/unittest/test_cython.py
new file mode 100644
index 0000000..d9ccd7a
--- /dev/null
+++ b/tests/python/unittest/test_cython.py
@@ -0,0 +1,75 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Debugging help
+# http://grapsus.net/blog/post/Low-level-Python-debugging-with-GDB
+
+from __future__ import print_function
+import sys
+import time
+from mxnet.base import _LIB
+
+try:
+ if sys.version_info >= (3, 0):
+ import mxnet.cython.cy3.mxcython as mxc
+ import mxnet.ndarray.cy3.ndarray as ndcy
+ import mxnet.symbol.cy3.symbol as symcy
+ else:
+ import mxnet.cython.cy2.mxcython as mxc
+ import mxnet.ndarray.cy2.ndarray as ndcy
+ import mxnet.symbol.cy2.symbol as symcy
+except:
+ # No cython found
+ print('Unable to load cython modules')
+ exit(1)
+
+def test_basic_cython():
+ print('ENTER test_basic_cython')
+ myclass = mxc.CythonTestClass()
+ for terms in 5, 9, 23, 177, 1111, 33333, 555555:
+ sys.stdout.write('{0:10} terms: '.format(terms))
+ mxc.print_pi(terms)
+ myclass.print_something('Something')
+ print('EXIT test_basic_cython')
+
+ # Test using a C++ class'
+ mxc.test_cpp_class()
+ mxc.test_perf(10, 1)
+
+
+def test_perf(count, make_c_call):
+ start = _LIB.TimeInMilliseconds()
+ foo = 0
+ i = 0
+ while i < count:
+ foo += i
+ if foo > count:
+ foo = 0
+ if make_c_call != 0:
+ _LIB.TrivialCPPCall(0)
+ i += 1
+ stop = _LIB.TimeInMilliseconds()
+ print("PYTHON: {} items took {} seconds".format(count, float(stop - start)/1000))
+
+if __name__ == '__main__':
+ # import nose
+ # nose.runmodule()
+ # test_perf(100000000, 0)
+ # mxc.test_perf(100000000, 0)
+ # test_perf(100000000, 1)
+ # mxc.test_perf(100000000, 1)
+ test_basic_cython()
diff --git a/python/mxnet/_cy3/__init__.py b/tools/cython/clean_cython.sh
old mode 100644
new mode 100755
similarity index 79%
copy from python/mxnet/_cy3/__init__.py
copy to tools/cython/clean_cython.sh
index 44dcca5..7f2d5f8
--- a/python/mxnet/_cy3/__init__.py
+++ b/tools/cython/clean_cython.sh
@@ -1,3 +1,4 @@
+#!/bin/sh
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -14,5 +15,6 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
-"""Namespace for cython generated modules for python3"""
+ROOTDIR=$(dirname $0)/..
+for i in $(find $ROOTDIR/python/mxnet -type f -name "*.so"); do rm -f $i; done
+for i in $(find $ROOTDIR/python/mxnet -type d -name "cython_debug"); do rm -rf $i; done
diff --git a/python/mxnet/_cy3/__init__.py b/tools/cython/cydb2
old mode 100644
new mode 100755
similarity index 68%
copy from python/mxnet/_cy3/__init__.py
copy to tools/cython/cydb2
index 44dcca5..5157a26
--- a/python/mxnet/_cy3/__init__.py
+++ b/tools/cython/cydb2
@@ -1,3 +1,4 @@
+#!/bin/bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -15,4 +16,12 @@
# specific language governing permissions and limitations
# under the License.
-"""Namespace for cython generated modules for python3"""
+ROOTDIR=$(realpath $(dirname $0))
+cd $ROOTDIR/../..
+if [ -d ../../cmake-build-debug/mxnet/cython/cy2 ]; then
+ cygdb ../../cmake-build-debug/mxnet/cython/cy2 -- $@
+elif [ -d ../cmake-build-debug/mxnet/cython/cy2 ]; then
+ cygdb ../cmake-build-debug/mxnet/cython/cy2 -- $@
+else
+ echo "$0: Don't know where to find cythoin debug info"
+fi
diff --git a/python/mxnet/_cy3/__init__.py b/tools/cython/cydb3
old mode 100644
new mode 100755
similarity index 68%
rename from python/mxnet/_cy3/__init__.py
rename to tools/cython/cydb3
index 44dcca5..62a346a
--- a/python/mxnet/_cy3/__init__.py
+++ b/tools/cython/cydb3
@@ -1,3 +1,4 @@
+#!/bin/bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -15,4 +16,12 @@
# specific language governing permissions and limitations
# under the License.
-"""Namespace for cython generated modules for python3"""
+ROOTDIR=$(realpath $(dirname $0))
+cd $ROOTDIR/../..
+if [ -d ../../cmake-build-debug/mxnet/cython/cy3 ]; then
+ cygdb ../../cmake-build-debug/mxnet/cython/cy3 -- $@
+elif [ -d ../cmake-build-debug/mxnet/cython/cy3 ]; then
+ cygdb ../cmake-build-debug/mxnet/cython/cy3 -- $@
+else
+ echo "$0: Don't know where to find cythoin debug info"
+fi
--
To stop receiving notification emails like this one, please contact
cjolivier01@apache.org.