You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by al...@apache.org on 2021/08/06 18:53:21 UTC
[arrow-datafusion] branch master updated: Expose
ExecutionContext.register_csv to the python bindings (#524)
This is an automated email from the ASF dual-hosted git repository.
alamb pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow-datafusion.git
The following commit(s) were added to refs/heads/master by this push:
new 5a7bbcc Expose ExecutionContext.register_csv to the python bindings (#524)
5a7bbcc is described below
commit 5a7bbccf9580897443aac85db6df04608a9d84d1
Author: Krisztián Szűcs <sz...@gmail.com>
AuthorDate: Fri Aug 6 20:53:14 2021 +0200
Expose ExecutionContext.register_csv to the python bindings (#524)
* Expose register_csv
* Validate delimiter
* Fix tests
* Pass schema
* unused imports
* add linting
* Update deps
* Restore venv
---
.github/workflows/python_test.yaml | 19 ++-
python/requirements.in | 10 +-
python/requirements.txt | 285 +++++++++++++++++++-----------------
python/src/context.rs | 45 ++++++
python/src/to_rust.rs | 9 ++
python/tests/generic.py | 13 +-
python/tests/test_math_functions.py | 16 +-
python/tests/test_pa_types.py | 5 +-
python/tests/test_sql.py | 65 +++++++-
9 files changed, 303 insertions(+), 164 deletions(-)
diff --git a/.github/workflows/python_test.yaml b/.github/workflows/python_test.yaml
index ebf5e9f..8d2eb85 100644
--- a/.github/workflows/python_test.yaml
+++ b/.github/workflows/python_test.yaml
@@ -41,18 +41,21 @@ jobs:
- uses: actions/setup-python@v2
with:
python-version: "3.9"
- - name: Install Python dependencies
- run: python -m pip install --upgrade pip setuptools wheel
- - name: Run tests
+ - name: Create Virtualenv
run: |
- cd python/
-
python -m venv venv
source venv/bin/activate
-
- pip install -r requirements.txt
+ pip install -r python/requirements.txt
+ - name: Run Linters
+ run: |
+ source venv/bin/activate
+ flake8 python
+ black --line-length 79 --check python
+ - name: Run tests
+ run: |
+ source venv/bin/activate
+ cd python
maturin develop
-
pytest -v .
env:
CARGO_HOME: "/home/runner/.cargo"
diff --git a/python/requirements.in b/python/requirements.in
index 5f145dc..7e54705 100644
--- a/python/requirements.in
+++ b/python/requirements.in
@@ -14,12 +14,14 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-maturin
-toml
-pyarrow
-pytest
+
black
+flake8
isort
+maturin
mypy
numpy
pandas
+pyarrow
+pytest
+toml
diff --git a/python/requirements.txt b/python/requirements.txt
index b7f0080..2176988 100644
--- a/python/requirements.txt
+++ b/python/requirements.txt
@@ -1,8 +1,8 @@
#
-# This file is autogenerated by pip-compile with python 3.9
+# This file is autogenerated by pip-compile with python 3.8
# To update, run:
#
-# pip-compile --generate-hashes
+# pip-compile --generate-hashes requirements.in
#
appdirs==1.4.4 \
--hash=sha256:7d5d0167b2b1ba821647616af46a749d1c653740dd0d2415100fe26e27afdf41 \
@@ -20,27 +20,35 @@ click==8.0.1 \
--hash=sha256:8c04c11192119b1ef78ea049e0a6f0463e4c48ef00a30160c704337586f3ad7a \
--hash=sha256:fba402a4a47334742d782209a7c79bc448911afe1149d07bdabdf480b3e2f4b6
# via black
+flake8==3.9.2 \
+ --hash=sha256:07528381786f2a6237b061f6e96610a4167b226cb926e2aa2b6b1d78057c576b \
+ --hash=sha256:bf8fd333346d844f616e8d47905ef3a3384edae6b4e9beb0c5101e25e3110907
+ # via -r requirements.in
iniconfig==1.1.1 \
--hash=sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3 \
--hash=sha256:bc3af051d7d14b2ee5ef9969666def0cd1a000e121eaea580d4a313df4b37f32
# via pytest
-isort==5.9.2 \
- --hash=sha256:eed17b53c3e7912425579853d078a0832820f023191561fcee9d7cae424e0813 \
- --hash=sha256:f65ce5bd4cbc6abdfbe29afc2f0245538ab358c14590912df638033f157d555e
+isort==5.9.3 \
+ --hash=sha256:9c2ea1e62d871267b78307fe511c0838ba0da28698c5732d54e2790bf3ba9899 \
+ --hash=sha256:e17d6e2b81095c9db0a03a8025a957f334d6ea30b26f9ec70805411e5c7c81f2
# via -r requirements.in
-maturin==0.11.1 \
- --hash=sha256:1d8a276b4c4ac74ecf9624ebc718982cdd0f86581d6338c877d7eb2833b89a13 \
- --hash=sha256:56b1dc8651a40d024a0ac59720ffeb61a41059fcd836f1742ad828b78650fc1a \
- --hash=sha256:70b35e77e60772002e279e87e936dd5467a7952b7ccc37054a1e478d3b25c279 \
- --hash=sha256:7b9f66a5425cf9f04276effbe31f1fea331a1bd742e1726effd72e48a98da0e1 \
- --hash=sha256:8921ab6dccde53625075b5d24d2e817bf36abaeef4387237fedb4e298e73e77d \
- --hash=sha256:a2a22ea9c8448796ce8078c7706623b212f2940ac79adaebe17000ada3d6647a \
- --hash=sha256:b322f36ee7ff67870fe6d0b5fcd41226a7eca05e6819ae812875ed5f2116038d \
- --hash=sha256:ba23ac9ca8d4a23ad794b9d966f09959dc5d511afd23992afc93b720dbe0f676 \
- --hash=sha256:c7dca1e2d8eabeb3dbd9b08a182be85621b2519a9968c728b9db73023bbdd823 \
- --hash=sha256:e1598a844fdc7b5093749feb0b373fb2f7545033bb1f00779cfbf173906e374a \
- --hash=sha256:e60308dd43eb5f763126d0651827683141b12878541c6ede008f77ef655d1343
+maturin==0.11.2 \
+ --hash=sha256:14afc56be161e52bad8211eba84d8f8e8fad90a86d1c1894a703a8801c6d81df \
+ --hash=sha256:2631a52d4936a550baa645921fe290749df3bc3c8be85255e650cb3b419dd445 \
+ --hash=sha256:27cbc80e9adb19957a4006274edddf6184e639b0c096dfa2f8fdd652401b2b69 \
+ --hash=sha256:520d3aaf4e0a8d520f9498ffd7f106641d5e5782cbf26d83da66c26c50b0a7c4 \
+ --hash=sha256:5c492ea891602088528485f6c257ffbe02143ea651b3287dcaccf3d542ebd6f7 \
+ --hash=sha256:8f520eccda84f028bce5556b2c8b6dce5eced105734fe8f645c72937f05cf054 \
+ --hash=sha256:99aed60b0427233e9d7005eea3d42887455c3be1e52a3b50246a86c4df215cd1 \
+ --hash=sha256:c13392b4e6913c35655bbbf62ed4f65a340c1501734ab172410b03068c4633f0 \
+ --hash=sha256:d2ed778b67317efd73f9ce580a6dd6569474e5a9bc3bfff4aa4d7e9a0bd9f888 \
+ --hash=sha256:de439e1572eb0711caeb6e4f4aec692829c7cc856b264861c73219b16be8b8d4 \
+ --hash=sha256:f14ddc72a3a37c72b8449704816e670f557e6d9148d35d342750dded7709eae4
# via -r requirements.in
+mccabe==0.6.1 \
+ --hash=sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42 \
+ --hash=sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f
+ # via flake8
mypy==0.910 \
--hash=sha256:088cd9c7904b4ad80bec811053272986611b84221835e079be5bcad029e79dd9 \
--hash=sha256:0aadfb2d3935988ec3815952e44058a3100499f5be5b28c34ac9d79f002a4a9a \
@@ -72,35 +80,35 @@ mypy-extensions==0.4.3 \
# via
# black
# mypy
-numpy==1.21.0 \
- --hash=sha256:1a784e8ff7ea2a32e393cc53eb0003eca1597c7ca628227e34ce34eb11645a0e \
- --hash=sha256:2ba579dde0563f47021dcd652253103d6fd66165b18011dce1a0609215b2791e \
- --hash=sha256:3537b967b350ad17633b35c2f4b1a1bbd258c018910b518c30b48c8e41272717 \
- --hash=sha256:3c40e6b860220ed862e8097b8f81c9af6d7405b723f4a7af24a267b46f90e461 \
- --hash=sha256:598fe100b2948465cf3ed64b1a326424b5e4be2670552066e17dfaa67246011d \
- --hash=sha256:620732f42259eb2c4642761bd324462a01cdd13dd111740ce3d344992dd8492f \
- --hash=sha256:709884863def34d72b183d074d8ba5cfe042bc3ff8898f1ffad0209161caaa99 \
- --hash=sha256:75579acbadbf74e3afd1153da6177f846212ea2a0cc77de53523ae02c9256513 \
- --hash=sha256:7c55407f739f0bfcec67d0df49103f9333edc870061358ac8a8c9e37ea02fcd2 \
- --hash=sha256:a1f2fb2da242568af0271455b89aee0f71e4e032086ee2b4c5098945d0e11cf6 \
- --hash=sha256:a290989cd671cd0605e9c91a70e6df660f73ae87484218e8285c6522d29f6e38 \
- --hash=sha256:ac4fd578322842dbda8d968e3962e9f22e862b6ec6e3378e7415625915e2da4d \
- --hash=sha256:ad09f55cc95ed8d80d8ab2052f78cc21cb231764de73e229140d81ff49d8145e \
- --hash=sha256:b9205711e5440954f861ceeea8f1b415d7dd15214add2e878b4d1cf2bcb1a914 \
- --hash=sha256:bba474a87496d96e61461f7306fba2ebba127bed7836212c360f144d1e72ac54 \
- --hash=sha256:bebab3eaf0641bba26039fb0b2c5bf9b99407924b53b1ea86e03c32c64ef5aef \
- --hash=sha256:cc367c86eb87e5b7c9592935620f22d13b090c609f1b27e49600cd033b529f54 \
- --hash=sha256:ccc6c650f8700ce1e3a77668bb7c43e45c20ac06ae00d22bdf6760b38958c883 \
- --hash=sha256:cf680682ad0a3bef56dae200dbcbac2d57294a73e5b0f9864955e7dd7c2c2491 \
- --hash=sha256:d2910d0a075caed95de1a605df00ee03b599de5419d0b95d55342e9a33ad1fb3 \
- --hash=sha256:d5caa946a9f55511e76446e170bdad1d12d6b54e17a2afe7b189112ed4412bb8 \
- --hash=sha256:d89b0dc7f005090e32bb4f9bf796e1dcca6b52243caf1803fdd2b748d8561f63 \
- --hash=sha256:d95d16204cd51ff1a1c8d5f9958ce90ae190be81d348b514f9be39f878b8044a \
- --hash=sha256:e4d5a86a5257843a18fb1220c5f1c199532bc5d24e849ed4b0289fb59fbd4d8f \
- --hash=sha256:e58ddb53a7b4959932f5582ac455ff90dcb05fac3f8dcc8079498d43afbbde6c \
- --hash=sha256:e80fe25cba41c124d04c662f33f6364909b985f2eb5998aaa5ae4b9587242cce \
- --hash=sha256:eda2829af498946c59d8585a9fd74da3f810866e05f8df03a86f70079c7531dd \
- --hash=sha256:fd0a359c1c17f00cb37de2969984a74320970e0ceef4808c32e00773b06649d9
+numpy==1.21.1 \
+ --hash=sha256:01721eefe70544d548425a07c80be8377096a54118070b8a62476866d5208e33 \
+ --hash=sha256:0318c465786c1f63ac05d7c4dbcecd4d2d7e13f0959b01b534ea1e92202235c5 \
+ --hash=sha256:05a0f648eb28bae4bcb204e6fd14603de2908de982e761a2fc78efe0f19e96e1 \
+ --hash=sha256:1412aa0aec3e00bc23fbb8664d76552b4efde98fb71f60737c83efbac24112f1 \
+ --hash=sha256:25b40b98ebdd272bc3020935427a4530b7d60dfbe1ab9381a39147834e985eac \
+ --hash=sha256:2d4d1de6e6fb3d28781c73fbde702ac97f03d79e4ffd6598b880b2d95d62ead4 \
+ --hash=sha256:38e8648f9449a549a7dfe8d8755a5979b45b3538520d1e735637ef28e8c2dc50 \
+ --hash=sha256:4a3d5fb89bfe21be2ef47c0614b9c9c707b7362386c9a3ff1feae63e0267ccb6 \
+ --hash=sha256:635e6bd31c9fb3d475c8f44a089569070d10a9ef18ed13738b03049280281267 \
+ --hash=sha256:73101b2a1fef16602696d133db402a7e7586654682244344b8329cdcbbb82172 \
+ --hash=sha256:791492091744b0fe390a6ce85cc1bf5149968ac7d5f0477288f78c89b385d9af \
+ --hash=sha256:7a708a79c9a9d26904d1cca8d383bf869edf6f8e7650d85dbc77b041e8c5a0f8 \
+ --hash=sha256:88c0b89ad1cc24a5efbb99ff9ab5db0f9a86e9cc50240177a571fbe9c2860ac2 \
+ --hash=sha256:8a326af80e86d0e9ce92bcc1e65c8ff88297de4fa14ee936cb2293d414c9ec63 \
+ --hash=sha256:8a92c5aea763d14ba9d6475803fc7904bda7decc2a0a68153f587ad82941fec1 \
+ --hash=sha256:91c6f5fc58df1e0a3cc0c3a717bb3308ff850abdaa6d2d802573ee2b11f674a8 \
+ --hash=sha256:95b995d0c413f5d0428b3f880e8fe1660ff9396dcd1f9eedbc311f37b5652e16 \
+ --hash=sha256:9749a40a5b22333467f02fe11edc98f022133ee1bfa8ab99bda5e5437b831214 \
+ --hash=sha256:978010b68e17150db8765355d1ccdd450f9fc916824e8c4e35ee620590e234cd \
+ --hash=sha256:9a513bd9c1551894ee3d31369f9b07460ef223694098cf27d399513415855b68 \
+ --hash=sha256:a75b4498b1e93d8b700282dc8e655b8bd559c0904b3910b144646dbbbc03e062 \
+ --hash=sha256:c6a2324085dd52f96498419ba95b5777e40b6bcbc20088fddb9e8cbb58885e8e \
+ --hash=sha256:d7a4aeac3b94af92a9373d6e77b37691b86411f9745190d2c351f410ab3a791f \
+ --hash=sha256:d9e7912a56108aba9b31df688a4c4f5cb0d9d3787386b87d504762b6754fbb1b \
+ --hash=sha256:dff4af63638afcc57a3dfb9e4b26d434a7a602d225b42d746ea7fe2edf1342fd \
+ --hash=sha256:e46ceaff65609b5399163de5893d8f2a82d3c77d5e56d976c8b5fb01faa6b671 \
+ --hash=sha256:f01f28075a92eede918b965e86e8f0ba7b7797a95aa8d35e1cc8821f5fc3ad6a \
+ --hash=sha256:fd7d7409fa643a91d0a05c7554dd68aa9c9bb16e186f6ccfe40d6e003156e33a
# via
# -r requirements.in
# pandas
@@ -109,26 +117,26 @@ packaging==21.0 \
--hash=sha256:7dc96269f53a4ccec5c0670940a4281106dd0bb343f47b7471f779df49c2fbe7 \
--hash=sha256:c86254f9220d55e31cc94d69bade760f0847da8000def4dfe1c6b872fd14ff14
# via pytest
-pandas==1.3.0 \
- --hash=sha256:08eeff3da6a188e24db7f292b39a8ca9e073bf841fbbeadb946b3ad5c19d843e \
- --hash=sha256:1ff13eed501e07e7fb26a4ea18a846b6e5d7de549b497025601fd9ccb7c1d123 \
- --hash=sha256:522bfea92f3ef6207cadc7428bda1e7605dae0383b8065030e7b5d0266717b48 \
- --hash=sha256:7897326cae660eee69d501cbfa950281a193fcf407393965e1bc07448e1cc35a \
- --hash=sha256:798675317d0e4863a92a9a6bc5bd2490b5f6fef8c17b95f29e2e33f28bef9eca \
- --hash=sha256:7d3cd2c99faa94d717ca00ea489264a291ad7209453dffbf059bfb7971fd3a61 \
- --hash=sha256:823737830364d0e2af8c3912a28ba971296181a07950873492ed94e12d28c405 \
- --hash=sha256:872aa91e0f9ca913046ab639d4181a899f5e592030d954d28c2529b88756a736 \
- --hash=sha256:88864c1e28353b958b1f30e4193818519624ad9a1776921622a6a2a016d5d807 \
- --hash=sha256:92835113a67cbd34747c198d41f09f4b63f6fe11ca5643baebc7ab1e30e89e95 \
- --hash=sha256:98efc2d4983d5bb47662fe2d97b2c81b91566cb08b266490918b9c7d74a5ef64 \
- --hash=sha256:b10d7910ae9d7920a5ff7816d794d99acbc361f7b16a0f017d4fa83ced8cb55e \
- --hash=sha256:c554e6c9cf2d5ea1aba5979cc837b3649539ced0e18ece186f055450c86622e2 \
- --hash=sha256:c746876cdd8380be0c3e70966d4566855901ac9aaa5e4b9ccaa5ca5311457d11 \
- --hash=sha256:c81b8d91e9ae861eb4406b4e0f8d4dabbc105b9c479b3d1e921fba1d35b5b62a \
- --hash=sha256:e6b75091fa54a53db3927b4d1bc997c23c5ba6f87acdfe1ee5a92c38c6b2ed6a \
- --hash=sha256:ed4fc66f23fe17c93a5d439230ca2d6b5f8eac7154198d327dbe8a16d98f3f10 \
- --hash=sha256:f058c786e7b0a9e7fa5e0b9f4422e0ccdd3bf3aa3053c18d77ed2a459bd9a45a \
- --hash=sha256:fe7a549d10ca534797095586883a5c17d140d606747591258869c56e14d1b457
+pandas==1.3.1 \
+ --hash=sha256:0c976e023ed580e60a82ccebdca8e1cc24d8b1fbb28175eb6521025c127dab66 \
+ --hash=sha256:114c6789d15862508900a25cb4cb51820bfdd8595ea306bab3b53cd19f990b65 \
+ --hash=sha256:1ee8418d0f936ff2216513aa03e199657eceb67690995d427a4a7ecd2e68f442 \
+ --hash=sha256:22f3fcc129fb482ef44e7df2a594f0bd514ac45aabe50da1a10709de1b0f9d84 \
+ --hash=sha256:23c7452771501254d2ae23e9e9dac88417de7e6eff3ce64ee494bb94dc88c300 \
+ --hash=sha256:341935a594db24f3ff07d1b34d1d231786aa9adfa84b76eab10bf42907c8aed3 \
+ --hash=sha256:45656cd59ae9745a1a21271a62001df58342b59c66d50754390066db500a8362 \
+ --hash=sha256:527c43311894aff131dea99cf418cd723bfd4f0bcf3c3da460f3b57e52a64da5 \
+ --hash=sha256:5c09a2538f0fddf3895070579082089ff4ae52b6cb176d8ec7a4dacf7e3676c1 \
+ --hash=sha256:5d9acfca191140a518779d1095036d842d5e5bc8e8ad8b5eaad1aff90fe1870d \
+ --hash=sha256:5ee927c70794e875a59796fab8047098aa59787b1be680717c141cd7873818ae \
+ --hash=sha256:7150039e78a81eddd9f5a05363a11cadf90a4968aac6f086fd83e66cf1c8d1d6 \
+ --hash=sha256:905fc3e0fcd86b0a9f1f97abee7d36894698d2592b22b859f08ea5a8fe3d3aab \
+ --hash=sha256:9d06661c6eb741ae633ee1c57e8c432bb4203024e263fe1a077fa3fda7817fdb \
+ --hash=sha256:9e1fe6722cbe27eb5891c1977bca62d456c19935352eea64d33956db46139364 \
+ --hash=sha256:be12d77f7e03c40a2466ed00ccd1a5f20a574d3c622fe1516037faa31aa448aa \
+ --hash=sha256:c28760932283d2c9f6fa5e53d2f77a514163b9e67fd0ee0879081be612567195 \
+ --hash=sha256:e323028ab192fcfe1e8999c012a0fa96d066453bb354c7e7a4a267b25e73d3c8 \
+ --hash=sha256:fdb3b33dde260b1766ea4d3c6b8fbf6799cee18d50a2a8bc534cf3550b7c819a
# via -r requirements.in
pathspec==0.9.0 \
--hash=sha256:7d15c4ddb0b5c802d161efc417ec1a2558ea2653c2e8ad9c19098201dc1c993a \
@@ -142,33 +150,44 @@ py==1.10.0 \
--hash=sha256:21b81bda15b66ef5e1a777a21c4dcd9c20ad3efd0b3f817e7a809035269e1bd3 \
--hash=sha256:3b80836aa6d1feeaa108e046da6423ab8f6ceda6468545ae8d02d9d58d18818a
# via pytest
-pyarrow==4.0.1 \
- --hash=sha256:04be0f7cb9090bd029b5b53bed628548fef569e5d0b5c6cd7f6d0106dbbc782d \
- --hash=sha256:0fde9c7a3d5d37f3fe5d18c4ed015e8f585b68b26d72a10d7012cad61afe43ff \
- --hash=sha256:11517f0b4f4acbab0c37c674b4d1aad3c3dfea0f6b1bb322e921555258101ab3 \
- --hash=sha256:150db335143edd00d3ec669c7c8167d401c4aa0a290749351c80bbf146892b2e \
- --hash=sha256:24040a20208e9b16ba7b284624ebfe67e40f5c40b5dc8d874da322ac0053f9d3 \
- --hash=sha256:33c457728a1ce825b80aa8c8ed573709f1efe72003d45fa6fdbb444de9cc0b74 \
- --hash=sha256:423cd6a14810f4e40cb76e13d4240040fc1594d69fe1c4f2c70be00ad512ade5 \
- --hash=sha256:5387db80c6a7b5598884bf4df3fc546b3373771ad614548b782e840b71704877 \
- --hash=sha256:5a76ec44af838862b23fb5cfc48765bc7978f7b58a181c96ad92856280de548b \
- --hash=sha256:5f2660f59dfcfd34adac7c08dc7f615920de703f191066ed6277628975f06878 \
- --hash=sha256:6b7bd8f5aa327cc32a1b9b02a76502851575f5edb110f93c59a45c70211a5618 \
- --hash=sha256:72cf3477538bd8504f14d6299a387cc335444f7a188f548096dfea9533551f02 \
- --hash=sha256:76b75a9cfc572e890a1e000fd532bdd2084ec3f1ee94ee51802a477913a21072 \
- --hash=sha256:a81adbfbe2f6528d4593b5a8962b2751838517401d14e9d4cab6787478802693 \
- --hash=sha256:a968375c66e505f72b421f5864a37f51aad5da61b6396fa283f956e9f2b2b923 \
- --hash=sha256:afd4f7c0a225a326d2c0039cdc8631b5e8be30f78f6b7a3e5ce741cf5dd81c72 \
- --hash=sha256:b05bdd513f045d43228247ef4d9269c88139788e2d566f4cb3e855e282ad0330 \
- --hash=sha256:c2733c9bcd00074ce5497dd0a7b8a10c91d3395ddce322d7021c7fdc4ea6f610 \
- --hash=sha256:d0f080b2d9720bec42624cb0df66f60ae66b84a2ccd1fe2c291322df915ac9db \
- --hash=sha256:dcd20ee0240a88772eeb5691102c276f5cdec79527fb3a0679af7f93f93cb4bd \
- --hash=sha256:e1351576877764fb4d5690e4721ce902e987c85f4ab081c70a34e1d24646586e \
- --hash=sha256:e44dfd7e61c9eb6dda59bc49ad69e77945f6d049185a517c130417e3ca0494d8 \
- --hash=sha256:ee3d87615876550fee9a523307dd4b00f0f44cf47a94a32a07793da307df31a0 \
- --hash=sha256:fa7b165cfa97158c1e6d15c68428317b4f4ae786d1dc2dbab43f1328c1eb43aa \
- --hash=sha256:fe976695318560a97c6d31bba828eeca28c44c6f6401005e54ba476a28ac0a10
+pyarrow==5.0.0 \
+ --hash=sha256:1832709281efefa4f199c639e9f429678286329860188e53beeda71750775923 \
+ --hash=sha256:1d9485741e497ccc516cb0a0c8f56e22be55aea815be185c3f9a681323b0e614 \
+ --hash=sha256:24e64ea33eed07441cc0e80c949e3a1b48211a1add8953268391d250f4d39922 \
+ --hash=sha256:2d26186ca9748a1fb89ae6c1fa04fb343a4279b53f118734ea8096f15d66c820 \
+ --hash=sha256:357605665fbefb573d40939b13a684c2490b6ed1ab4a5de8dd246db4ab02e5a4 \
+ --hash=sha256:4341ac0f552dc04c450751e049976940c7f4f8f2dae03685cc465ebe0a61e231 \
+ --hash=sha256:456a4488ae810a0569d1adf87dbc522bcc9a0e4a8d1809b934ca28c163d8edce \
+ --hash=sha256:4d8adda1892ef4553c4804af7f67cce484f4d6371564e2d8374b8e2bc85293e2 \
+ --hash=sha256:53e550dec60d1ab86cba3afa1719dc179a8bc9632a0e50d9fe91499cf0a7f2bc \
+ --hash=sha256:5c0d1b68e67bb334a5af0cecdf9b6a702aaa4cc259c5cbb71b25bbed40fcedaf \
+ --hash=sha256:601b0aabd6fb066429e706282934d4d8d38f53bdb8d82da9576be49f07eedf5c \
+ --hash=sha256:64f30aa6b28b666a925d11c239344741850eb97c29d3aa0f7187918cf82494f7 \
+ --hash=sha256:6e1f0e4374061116f40e541408a8a170c170d0a070b788717e18165ebfdd2a54 \
+ --hash=sha256:6e937ce4a40ea0cc7896faff96adecadd4485beb53fbf510b46858e29b2e75ae \
+ --hash=sha256:7560332e5846f0e7830b377c14c93624e24a17f91c98f0b25dafb0ca1ea6ba02 \
+ --hash=sha256:7c4edd2bacee3eea6c8c28bddb02347f9d41a55ec9692c71c6de6e47c62a7f0d \
+ --hash=sha256:99c8b0f7e2ce2541dd4c0c0101d9944bb8e592ae3295fe7a2f290ab99222666d \
+ --hash=sha256:9e04d3621b9f2f23898eed0d044203f66c156d880f02c5534a7f9947ebb1a4af \
+ --hash=sha256:b1453c2411b5062ba6bf6832dbc4df211ad625f678c623a2ee177aee158f199b \
+ --hash=sha256:b3115df938b8d7a7372911a3cb3904196194bcea8bb48911b4b3eafee3ab8d90 \
+ --hash=sha256:b6387d2058d95fa48ccfedea810a768187affb62f4a3ef6595fa30bf9d1a65cf \
+ --hash=sha256:bbe2e439bec2618c74a3bb259700c8a7353dc2ea0c5a62686b6cf04a50ab1e0d \
+ --hash=sha256:c3fc856f107ca2fb3c9391d7ea33bbb33f3a1c2b4a0e2b41f7525c626214cc03 \
+ --hash=sha256:c5493d2414d0d690a738aac8dd6d38518d1f9b870e52e24f89d8d7eb3afd4161 \
+ --hash=sha256:e9ec80f4a77057498cf4c5965389e42e7f6a618b6859e6dd615e57505c9167a6 \
+ --hash=sha256:ed135a99975380c27077f9d0e210aea8618ed9fadcec0e71f8a3190939557afe \
+ --hash=sha256:f4db312e9ba80e730cefcae0a05b63ea5befc7634c28df56682b628ad8e1c25c \
+ --hash=sha256:ff21711f6ff3b0bc90abc8ca8169e676faeb2401ddc1a0bc1c7dc181708a3406
# via -r requirements.in
+pycodestyle==2.7.0 \
+ --hash=sha256:514f76d918fcc0b55c6680472f0a37970994e07bbb80725808c17089be302068 \
+ --hash=sha256:c389c1d06bf7904078ca03399a4816f974a1d590090fecea0c63ec26ebaf1cef
+ # via flake8
+pyflakes==2.3.1 \
+ --hash=sha256:7893783d01b8a89811dd72d7dfd4d84ff098e5eed95cfa8905b22bbffe52efc3 \
+ --hash=sha256:f5bc8ecabc05bb9d291eb5203d6810b49040f6ff446a756326104746cc00c1db
+ # via flake8
pyparsing==2.4.7 \
--hash=sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1 \
--hash=sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b
@@ -185,48 +204,40 @@ pytz==2021.1 \
--hash=sha256:83a4a90894bf38e243cf052c8b58f381bfe9a7a483f6a9cab140bc7f702ac4da \
--hash=sha256:eb10ce3e7736052ed3623d49975ce333bcd712c7bb19a58b9e2089d4057d0798
# via pandas
-regex==2021.7.6 \
- --hash=sha256:0eb2c6e0fcec5e0f1d3bcc1133556563222a2ffd2211945d7b1480c1b1a42a6f \
- --hash=sha256:15dddb19823f5147e7517bb12635b3c82e6f2a3a6b696cc3e321522e8b9308ad \
- --hash=sha256:173bc44ff95bc1e96398c38f3629d86fa72e539c79900283afa895694229fe6a \
- --hash=sha256:1c78780bf46d620ff4fff40728f98b8afd8b8e35c3efd638c7df67be2d5cddbf \
- --hash=sha256:2366fe0479ca0e9afa534174faa2beae87847d208d457d200183f28c74eaea59 \
- --hash=sha256:2bceeb491b38225b1fee4517107b8491ba54fba77cf22a12e996d96a3c55613d \
- --hash=sha256:2ddeabc7652024803666ea09f32dd1ed40a0579b6fbb2a213eba590683025895 \
- --hash=sha256:2fe5e71e11a54e3355fa272137d521a40aace5d937d08b494bed4529964c19c4 \
- --hash=sha256:319eb2a8d0888fa6f1d9177705f341bc9455a2c8aca130016e52c7fe8d6c37a3 \
- --hash=sha256:3f5716923d3d0bfb27048242a6e0f14eecdb2e2a7fac47eda1d055288595f222 \
- --hash=sha256:422dec1e7cbb2efbbe50e3f1de36b82906def93ed48da12d1714cabcd993d7f0 \
- --hash=sha256:4c9c3155fe74269f61e27617529b7f09552fbb12e44b1189cebbdb24294e6e1c \
- --hash=sha256:4f64fc59fd5b10557f6cd0937e1597af022ad9b27d454e182485f1db3008f417 \
- --hash=sha256:564a4c8a29435d1f2256ba247a0315325ea63335508ad8ed938a4f14c4116a5d \
- --hash=sha256:59506c6e8bd9306cd8a41511e32d16d5d1194110b8cfe5a11d102d8b63cf945d \
- --hash=sha256:598c0a79b4b851b922f504f9f39a863d83ebdfff787261a5ed061c21e67dd761 \
- --hash=sha256:59c00bb8dd8775473cbfb967925ad2c3ecc8886b3b2d0c90a8e2707e06c743f0 \
- --hash=sha256:6110bab7eab6566492618540c70edd4d2a18f40ca1d51d704f1d81c52d245026 \
- --hash=sha256:6afe6a627888c9a6cfbb603d1d017ce204cebd589d66e0703309b8048c3b0854 \
- --hash=sha256:791aa1b300e5b6e5d597c37c346fb4d66422178566bbb426dd87eaae475053fb \
- --hash=sha256:8394e266005f2d8c6f0bc6780001f7afa3ef81a7a2111fa35058ded6fce79e4d \
- --hash=sha256:875c355360d0f8d3d827e462b29ea7682bf52327d500a4f837e934e9e4656068 \
- --hash=sha256:89e5528803566af4df368df2d6f503c84fbfb8249e6631c7b025fe23e6bd0cde \
- --hash=sha256:99d8ab206a5270c1002bfcf25c51bf329ca951e5a169f3b43214fdda1f0b5f0d \
- --hash=sha256:9a854b916806c7e3b40e6616ac9e85d3cdb7649d9e6590653deb5b341a736cec \
- --hash=sha256:b85ac458354165405c8a84725de7bbd07b00d9f72c31a60ffbf96bb38d3e25fa \
- --hash=sha256:bc84fb254a875a9f66616ed4538542fb7965db6356f3df571d783f7c8d256edd \
- --hash=sha256:c92831dac113a6e0ab28bc98f33781383fe294df1a2c3dfd1e850114da35fd5b \
- --hash=sha256:cbe23b323988a04c3e5b0c387fe3f8f363bf06c0680daf775875d979e376bd26 \
- --hash=sha256:ccb3d2190476d00414aab36cca453e4596e8f70a206e2aa8db3d495a109153d2 \
- --hash=sha256:d8bbce0c96462dbceaa7ac4a7dfbbee92745b801b24bce10a98d2f2b1ea9432f \
- --hash=sha256:db2b7df831c3187a37f3bb80ec095f249fa276dbe09abd3d35297fc250385694 \
- --hash=sha256:e586f448df2bbc37dfadccdb7ccd125c62b4348cb90c10840d695592aa1b29e0 \
- --hash=sha256:e5983c19d0beb6af88cb4d47afb92d96751fb3fa1784d8785b1cdf14c6519407 \
- --hash=sha256:e6a1e5ca97d411a461041d057348e578dc344ecd2add3555aedba3b408c9f874 \
- --hash=sha256:eaf58b9e30e0e546cdc3ac06cf9165a1ca5b3de8221e9df679416ca667972035 \
- --hash=sha256:ed693137a9187052fc46eedfafdcb74e09917166362af4cc4fddc3b31560e93d \
- --hash=sha256:edd1a68f79b89b0c57339bce297ad5d5ffcc6ae7e1afdb10f1947706ed066c9c \
- --hash=sha256:f080248b3e029d052bf74a897b9d74cfb7643537fbde97fe8225a6467fb559b5 \
- --hash=sha256:f9392a4555f3e4cb45310a65b403d86b589adc773898c25a39184b1ba4db8985 \
- --hash=sha256:f98dc35ab9a749276f1a4a38ab3e0e2ba1662ce710f6530f5b0a6656f1c32b58
+regex==2021.8.3 \
+ --hash=sha256:026beb631097a4a3def7299aa5825e05e057de3c6d72b139c37813bfa351274b \
+ --hash=sha256:14caacd1853e40103f59571f169704367e79fb78fac3d6d09ac84d9197cadd16 \
+ --hash=sha256:16d9eaa8c7e91537516c20da37db975f09ac2e7772a0694b245076c6d68f85da \
+ --hash=sha256:18fdc51458abc0a974822333bd3a932d4e06ba2a3243e9a1da305668bd62ec6d \
+ --hash=sha256:28e8af338240b6f39713a34e337c3813047896ace09d51593d6907c66c0708ba \
+ --hash=sha256:3835de96524a7b6869a6c710b26c90e94558c31006e96ca3cf6af6751b27dca1 \
+ --hash=sha256:3905c86cc4ab6d71635d6419a6f8d972cab7c634539bba6053c47354fd04452c \
+ --hash=sha256:3c09d88a07483231119f5017904db8f60ad67906efac3f1baa31b9b7f7cca281 \
+ --hash=sha256:4551728b767f35f86b8e5ec19a363df87450c7376d7419c3cac5b9ceb4bce576 \
+ --hash=sha256:459bbe342c5b2dec5c5223e7c363f291558bc27982ef39ffd6569e8c082bdc83 \
+ --hash=sha256:4f421e3cdd3a273bace013751c345f4ebeef08f05e8c10757533ada360b51a39 \
+ --hash=sha256:577737ec3d4c195c4aef01b757905779a9e9aee608fa1cf0aec16b5576c893d3 \
+ --hash=sha256:57fece29f7cc55d882fe282d9de52f2f522bb85290555b49394102f3621751ee \
+ --hash=sha256:7976d410e42be9ae7458c1816a416218364e06e162b82e42f7060737e711d9ce \
+ --hash=sha256:85f568892422a0e96235eb8ea6c5a41c8ccbf55576a2260c0160800dbd7c4f20 \
+ --hash=sha256:8764a78c5464ac6bde91a8c87dd718c27c1cabb7ed2b4beaf36d3e8e390567f9 \
+ --hash=sha256:8935937dad2c9b369c3d932b0edbc52a62647c2afb2fafc0c280f14a8bf56a6a \
+ --hash=sha256:8fe58d9f6e3d1abf690174fd75800fda9bdc23d2a287e77758dc0e8567e38ce6 \
+ --hash=sha256:937b20955806381e08e54bd9d71f83276d1f883264808521b70b33d98e4dec5d \
+ --hash=sha256:9569da9e78f0947b249370cb8fadf1015a193c359e7e442ac9ecc585d937f08d \
+ --hash=sha256:a3b73390511edd2db2d34ff09aa0b2c08be974c71b4c0505b4a048d5dc128c2b \
+ --hash=sha256:a4eddbe2a715b2dd3849afbdeacf1cc283160b24e09baf64fa5675f51940419d \
+ --hash=sha256:a5c6dbe09aff091adfa8c7cfc1a0e83fdb8021ddb2c183512775a14f1435fe16 \
+ --hash=sha256:b63e3571b24a7959017573b6455e05b675050bbbea69408f35f3cb984ec54363 \
+ --hash=sha256:bb350eb1060591d8e89d6bac4713d41006cd4d479f5e11db334a48ff8999512f \
+ --hash=sha256:bf6d987edd4a44dd2fa2723fca2790f9442ae4de2c8438e53fcb1befdf5d823a \
+ --hash=sha256:bfa6a679410b394600eafd16336b2ce8de43e9b13f7fb9247d84ef5ad2b45e91 \
+ --hash=sha256:c856ec9b42e5af4fe2d8e75970fcc3a2c15925cbcc6e7a9bcb44583b10b95e80 \
+ --hash=sha256:cea56288eeda8b7511d507bbe7790d89ae7049daa5f51ae31a35ae3c05408531 \
+ --hash=sha256:ea212df6e5d3f60341aef46401d32fcfded85593af1d82b8b4a7a68cd67fdd6b \
+ --hash=sha256:f35567470ee6dbfb946f069ed5f5615b40edcbb5f1e6e1d3d2b114468d505fc6 \
+ --hash=sha256:fbc20975eee093efa2071de80df7f972b7b35e560b213aafabcec7c0bd00bd8c \
+ --hash=sha256:ff4a8ad9638b7ca52313d8732f37ecd5fd3c8e3aff10a8ccb93176fd5b3812f6
# via black
six==1.16.0 \
--hash=sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926 \
@@ -240,9 +251,9 @@ toml==0.10.2 \
# maturin
# mypy
# pytest
-tomli==1.0.4 \
- --hash=sha256:0713b16ff91df8638a6a694e295c8159ab35ba93e3424a626dd5226d386057be \
- --hash=sha256:be670d0d8d7570fd0ea0113bd7bb1ba3ac6706b4de062cc4c952769355c9c268
+tomli==1.2.0 \
+ --hash=sha256:056f0376bf5a6b182c513f9582c1e5b0487265eb6c48842b69aa9ca1cd5f640a \
+ --hash=sha256:d60e681734099207a6add7a10326bc2ddd1fdc36c1b0f547d00ef73ac63739c2
# via black
typing-extensions==3.10.0.0 \
--hash=sha256:0ac0f89795dd19de6b97debb0c6af1c70987fd80a2d62d1958f7e56fcc31b497 \
diff --git a/python/src/context.rs b/python/src/context.rs
index 14ef0f7..9acc14a 100644
--- a/python/src/context.rs
+++ b/python/src/context.rs
@@ -15,16 +15,19 @@
// specific language governing permissions and limitations
// under the License.
+use std::path::PathBuf;
use std::{collections::HashSet, sync::Arc};
use rand::distributions::Alphanumeric;
use rand::Rng;
+use pyo3::exceptions::PyValueError;
use pyo3::prelude::*;
use datafusion::arrow::record_batch::RecordBatch;
use datafusion::datasource::MemTable;
use datafusion::execution::context::ExecutionContext as _ExecutionContext;
+use datafusion::prelude::CsvReadOptions;
use crate::dataframe;
use crate::errors;
@@ -97,6 +100,48 @@ impl ExecutionContext {
Ok(())
}
+ #[args(
+ schema = "None",
+ has_header = "true",
+ delimiter = "\",\"",
+ schema_infer_max_records = "1000",
+ file_extension = "\".csv\""
+ )]
+ fn register_csv(
+ &mut self,
+ name: &str,
+ path: PathBuf,
+ schema: Option<&PyAny>,
+ has_header: bool,
+ delimiter: &str,
+ schema_infer_max_records: usize,
+ file_extension: &str,
+ ) -> PyResult<()> {
+ let path = path
+ .to_str()
+ .ok_or(PyValueError::new_err("Unable to convert path to a string"))?;
+ let schema = match schema {
+ Some(s) => Some(to_rust::to_rust_schema(s)?),
+ None => None,
+ };
+ let delimiter = delimiter.as_bytes();
+ if delimiter.len() != 1 {
+ return Err(PyValueError::new_err(
+ "Delimiter must be a single character",
+ ));
+ }
+
+ let mut options = CsvReadOptions::new()
+ .has_header(has_header)
+ .delimiter(delimiter[0])
+ .schema_infer_max_records(schema_infer_max_records)
+ .file_extension(file_extension);
+ options.schema = schema.as_ref();
+
+ errors::wrap(self.ctx.register_csv(name, path, options))?;
+ Ok(())
+ }
+
fn register_udf(
&mut self,
name: &str,
diff --git a/python/src/to_rust.rs b/python/src/to_rust.rs
index e7957ec..7977fe4 100644
--- a/python/src/to_rust.rs
+++ b/python/src/to_rust.rs
@@ -15,6 +15,7 @@
// specific language governing permissions and limitations
// under the License.
+use std::convert::TryFrom;
use std::sync::Arc;
use datafusion::arrow::{
@@ -111,3 +112,11 @@ pub fn to_rust_scalar(ob: &PyAny) -> PyResult<ScalarValue> {
}
})
}
+
+pub fn to_rust_schema(ob: &PyAny) -> PyResult<Schema> {
+ let c_schema = ffi::FFI_ArrowSchema::empty();
+ let c_schema_ptr = &c_schema as *const ffi::FFI_ArrowSchema;
+ ob.call_method1("_export_to_c", (c_schema_ptr as uintptr_t,))?;
+ let schema = Schema::try_from(&c_schema).map_err(errors::DataFusionError::from)?;
+ Ok(schema)
+}
diff --git a/python/tests/generic.py b/python/tests/generic.py
index 5871c5e..8d5adaa 100644
--- a/python/tests/generic.py
+++ b/python/tests/generic.py
@@ -19,6 +19,7 @@ import datetime
import numpy as np
import pyarrow as pa
+import pyarrow.csv
import pyarrow.parquet as pq
# used to write parquet files
@@ -49,7 +50,9 @@ def data_datetime(f):
datetime.datetime.now() - datetime.timedelta(days=1),
datetime.datetime.now() + datetime.timedelta(days=1),
]
- return pa.array(data, type=pa.timestamp(f), mask=np.array([False, True, False]))
+ return pa.array(
+ data, type=pa.timestamp(f), mask=np.array([False, True, False])
+ )
def data_date32():
@@ -58,7 +61,9 @@ def data_date32():
datetime.date(1980, 1, 1),
datetime.date(2030, 1, 1),
]
- return pa.array(data, type=pa.date32(), mask=np.array([False, True, False]))
+ return pa.array(
+ data, type=pa.date32(), mask=np.array([False, True, False])
+ )
def data_timedelta(f):
@@ -67,7 +72,9 @@ def data_timedelta(f):
datetime.timedelta(days=1),
datetime.timedelta(seconds=1),
]
- return pa.array(data, type=pa.duration(f), mask=np.array([False, True, False]))
+ return pa.array(
+ data, type=pa.duration(f), mask=np.array([False, True, False])
+ )
def data_binary_other():
diff --git a/python/tests/test_math_functions.py b/python/tests/test_math_functions.py
index cb03753..98656b8 100644
--- a/python/tests/test_math_functions.py
+++ b/python/tests/test_math_functions.py
@@ -26,7 +26,9 @@ from datafusion import functions as f
def df():
ctx = ExecutionContext()
# create a RecordBatch and a new DataFrame from it
- batch = pa.RecordBatch.from_arrays([pa.array([0.1, -0.7, 0.55])], names=["value"])
+ batch = pa.RecordBatch.from_arrays(
+ [pa.array([0.1, -0.7, 0.55])], names=["value"]
+ )
return ctx.create_dataframe([[batch]])
@@ -56,7 +58,13 @@ def test_math_functions(df):
np.testing.assert_array_almost_equal(result.column(4), np.arcsin(values))
np.testing.assert_array_almost_equal(result.column(5), np.arccos(values))
np.testing.assert_array_almost_equal(result.column(6), np.exp(values))
- np.testing.assert_array_almost_equal(result.column(7), np.log(values + 1.0))
- np.testing.assert_array_almost_equal(result.column(8), np.log2(values + 1.0))
- np.testing.assert_array_almost_equal(result.column(9), np.log10(values + 1.0))
+ np.testing.assert_array_almost_equal(
+ result.column(7), np.log(values + 1.0)
+ )
+ np.testing.assert_array_almost_equal(
+ result.column(8), np.log2(values + 1.0)
+ )
+ np.testing.assert_array_almost_equal(
+ result.column(9), np.log10(values + 1.0)
+ )
np.testing.assert_array_less(result.column(10), np.ones_like(values))
diff --git a/python/tests/test_pa_types.py b/python/tests/test_pa_types.py
index 069343f..04f6110 100644
--- a/python/tests/test_pa_types.py
+++ b/python/tests/test_pa_types.py
@@ -19,8 +19,8 @@ import pyarrow as pa
def test_type_ids():
- """having this fixed is very important because internally we rely on this id to parse from
- python"""
+ # Having this fixed is very important because internally we rely on this id
+ # to parse from python
for idx, arrow_type in [
(0, pa.null()),
(1, pa.bool_()),
@@ -47,5 +47,4 @@ def test_type_ids():
(34, pa.large_utf8()),
(35, pa.large_binary()),
]:
-
assert idx == arrow_type.id
diff --git a/python/tests/test_sql.py b/python/tests/test_sql.py
index 62d6c09..669f640 100644
--- a/python/tests/test_sql.py
+++ b/python/tests/test_sql.py
@@ -18,8 +18,8 @@
import numpy as np
import pyarrow as pa
import pytest
-from datafusion import ExecutionContext
+from datafusion import ExecutionContext
from . import generic as helpers
@@ -33,12 +33,63 @@ def test_no_table(ctx):
ctx.sql("SELECT a FROM b").collect()
-def test_register(ctx, tmp_path):
+def test_register_csv(ctx, tmp_path):
+ path = tmp_path / "test.csv"
+
+ table = pa.Table.from_arrays(
+ [
+ [1, 2, 3, 4],
+ ["a", "b", "c", "d"],
+ [1.1, 2.2, 3.3, 4.4],
+ ],
+ names=["int", "str", "float"],
+ )
+ pa.csv.write_csv(table, path)
+
+ ctx.register_csv("csv", path)
+ ctx.register_csv("csv1", str(path))
+ ctx.register_csv(
+ "csv2",
+ path,
+ has_header=True,
+ delimiter=",",
+ schema_infer_max_records=10,
+ )
+ alternative_schema = pa.schema(
+ [
+ ("some_int", pa.int16()),
+ ("some_bytes", pa.string()),
+ ("some_floats", pa.float32()),
+ ]
+ )
+ ctx.register_csv("csv3", path, schema=alternative_schema)
+
+ assert ctx.tables() == {"csv", "csv1", "csv2", "csv3"}
+
+ for table in ["csv", "csv1", "csv2"]:
+ result = ctx.sql(f"SELECT COUNT(int) FROM {table}").collect()
+ result = pa.Table.from_batches(result)
+ assert result.to_pydict() == {"COUNT(int)": [4]}
+
+ result = ctx.sql("SELECT * FROM csv3").collect()
+ result = pa.Table.from_batches(result)
+ assert result.schema == alternative_schema
+
+ with pytest.raises(
+ ValueError, match="Delimiter must be a single character"
+ ):
+ ctx.register_csv("csv4", path, delimiter="wrong")
+
+
+def test_register_parquet(ctx, tmp_path):
path = helpers.write_parquet(tmp_path / "a.parquet", helpers.data())
ctx.register_parquet("t", path)
-
assert ctx.tables() == {"t"}
+ result = ctx.sql("SELECT COUNT(a) FROM t").collect()
+ result = pa.Table.from_batches(result)
+ assert result.to_pydict() == {"COUNT(a)": [100]}
+
def test_execute(ctx, tmp_path):
data = [1, 1, 2, 2, 3, 11, 12]
@@ -112,7 +163,9 @@ def test_cast(ctx, tmp_path):
"float",
]
- select = ", ".join([f"CAST(9 AS {t}) AS A{i}" for i, t in enumerate(valid_types)])
+ select = ", ".join(
+ [f"CAST(9 AS {t}) AS A{i}" for i, t in enumerate(valid_types)]
+ )
# can execute, which implies that we can cast
ctx.sql(f"SELECT {select} FROM t").collect()
@@ -141,7 +194,9 @@ def test_udf(
ctx, tmp_path, fn, input_types, output_type, input_values, expected_values
):
# write to disk
- path = helpers.write_parquet(tmp_path / "a.parquet", pa.array(input_values))
+ path = helpers.write_parquet(
+ tmp_path / "a.parquet", pa.array(input_values)
+ )
ctx.register_parquet("t", path)
ctx.register_udf("udf", fn, input_types, output_type)