You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by bl...@apache.org on 2021/10/03 17:17:50 UTC
[iceberg] branch master updated: Python: Setup new python library
module structure (#3184)
This is an automated email from the ASF dual-hosted git repository.
blue pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iceberg.git
The following commit(s) were added to refs/heads/master by this push:
new 187f503 Python: Setup new python library module structure (#3184)
187f503 is described below
commit 187f503f50861587f09a428569230844d7a2de38
Author: jun-he <ju...@users.noreply.github.com>
AuthorDate: Sun Oct 3 10:17:40 2021 -0700
Python: Setup new python library module structure (#3184)
---
.github/workflows/python-ci.yml | 8 +-
.../{python-ci.yml => python-legacy-ci.yml} | 6 +-
.gitignore | 1 +
python/LICENSE | 204 +++++++++++++++++++++
.../workflows/python-ci.yml => python/MANIFEST.in | 33 +---
python/NOTICE | 8 +
python/README.md | 43 +++++
.../python-ci.yml => python/pyproject.toml | 38 +---
python/setup.cfg | 47 +++++
.github/workflows/python-ci.yml => python/setup.py | 42 ++---
.../src/iceberg/__init__.py | 33 ----
.../src/iceberg/catalog/__init__.py | 33 ----
.../src/iceberg/io/__init__.py | 33 ----
.../src/iceberg/table/__init__.py | 33 ----
.../src/iceberg/utils/__init__.py | 33 ----
python/src/iceberg/utils/bin_packing.py | 81 ++++++++
.../python-ci.yml => python/tests/__init__.py | 33 ----
.../tests/utils/__init__.py | 33 ----
python/tests/utils/test_bin_packing.py | 50 +++++
python/tox.ini | 118 ++++++++++++
20 files changed, 577 insertions(+), 333 deletions(-)
diff --git a/.github/workflows/python-ci.yml b/.github/workflows/python-ci.yml
index 0c5a682..a274f10 100644
--- a/.github/workflows/python-ci.yml
+++ b/.github/workflows/python-ci.yml
@@ -22,11 +22,11 @@ on:
push:
paths:
- '.github/workflows/python-ci.yml'
- - 'python_legacy/**'
+ - 'python/**'
pull_request:
paths:
- '.github/workflows/python-ci.yml'
- - 'python_legacy/**'
+ - 'python/**'
jobs:
tox:
@@ -40,10 +40,10 @@ jobs:
- uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python }}
- - working-directory: ./python_legacy
+ - working-directory: ./python
run: |
pip install -e .[dev]
pip install -U tox-gh-actions
- - working-directory: ./python_legacy
+ - working-directory: ./python
run: tox
diff --git a/.github/workflows/python-ci.yml b/.github/workflows/python-legacy-ci.yml
similarity index 91%
copy from .github/workflows/python-ci.yml
copy to .github/workflows/python-legacy-ci.yml
index 0c5a682..8004d73 100644
--- a/.github/workflows/python-ci.yml
+++ b/.github/workflows/python-legacy-ci.yml
@@ -17,15 +17,15 @@
# under the License.
#
-name: "Python CI"
+name: "Python Legacy CI"
on:
push:
paths:
- - '.github/workflows/python-ci.yml'
+ - '.github/workflows/python-legacy-ci.yml'
- 'python_legacy/**'
pull_request:
paths:
- - '.github/workflows/python-ci.yml'
+ - '.github/workflows/python-legacy-ci.yml'
- 'python_legacy/**'
jobs:
diff --git a/.gitignore b/.gitignore
index 6c06d4a..777b0b1 100644
--- a/.gitignore
+++ b/.gitignore
@@ -61,3 +61,4 @@ derby.log
# Python stuff
python_legacy/.mypy_cache/
+python/.mypy_cache/
diff --git a/python/LICENSE b/python/LICENSE
new file mode 100644
index 0000000..7868b9b
--- /dev/null
+++ b/python/LICENSE
@@ -0,0 +1,204 @@
+
+ Apache License
+ Version 2.0, January 2004
+ http://www.apache.org/licenses/
+
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+ 1. Definitions.
+
+ "License" shall mean the terms and conditions for use, reproduction,
+ and distribution as defined by Sections 1 through 9 of this document.
+
+ "Licensor" shall mean the copyright owner or entity authorized by
+ the copyright owner that is granting the License.
+
+ "Legal Entity" shall mean the union of the acting entity and all
+ other entities that control, are controlled by, or are under common
+ control with that entity. For the purposes of this definition,
+ "control" means (i) the power, direct or indirect, to cause the
+ direction or management of such entity, whether by contract or
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
+ outstanding shares, or (iii) beneficial ownership of such entity.
+
+ "You" (or "Your") shall mean an individual or Legal Entity
+ exercising permissions granted by this License.
+
+ "Source" form shall mean the preferred form for making modifications,
+ including but not limited to software source code, documentation
+ source, and configuration files.
+
+ "Object" form shall mean any form resulting from mechanical
+ transformation or translation of a Source form, including but
+ not limited to compiled object code, generated documentation,
+ and conversions to other media types.
+
+ "Work" shall mean the work of authorship, whether in Source or
+ Object form, made available under the License, as indicated by a
+ copyright notice that is included in or attached to the work
+ (an example is provided in the Appendix below).
+
+ "Derivative Works" shall mean any work, whether in Source or Object
+ form, that is based on (or derived from) the Work and for which the
+ editorial revisions, annotations, elaborations, or other modifications
+ represent, as a whole, an original work of authorship. For the purposes
+ of this License, Derivative Works shall not include works that remain
+ separable from, or merely link (or bind by name) to the interfaces of,
+ the Work and Derivative Works thereof.
+
+ "Contribution" shall mean any work of authorship, including
+ the original version of the Work and any modifications or additions
+ to that Work or Derivative Works thereof, that is intentionally
+ submitted to Licensor for inclusion in the Work by the copyright owner
+ or by an individual or Legal Entity authorized to submit on behalf of
+ the copyright owner. For the purposes of this definition, "submitted"
+ means any form of electronic, verbal, or written communication sent
+ to the Licensor or its representatives, including but not limited to
+ communication on electronic mailing lists, source code control systems,
+ and issue tracking systems that are managed by, or on behalf of, the
+ Licensor for the purpose of discussing and improving the Work, but
+ excluding communication that is conspicuously marked or otherwise
+ designated in writing by the copyright owner as "Not a Contribution."
+
+ "Contributor" shall mean Licensor and any individual or Legal Entity
+ on behalf of whom a Contribution has been received by Licensor and
+ subsequently incorporated within the Work.
+
+ 2. Grant of Copyright License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ copyright license to reproduce, prepare Derivative Works of,
+ publicly display, publicly perform, sublicense, and distribute the
+ Work and such Derivative Works in Source or Object form.
+
+ 3. Grant of Patent License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ (except as stated in this section) patent license to make, have made,
+ use, offer to sell, sell, import, and otherwise transfer the Work,
+ where such license applies only to those patent claims licensable
+ by such Contributor that are necessarily infringed by their
+ Contribution(s) alone or by combination of their Contribution(s)
+ with the Work to which such Contribution(s) was submitted. If You
+ institute patent litigation against any entity (including a
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
+ or a Contribution incorporated within the Work constitutes direct
+ or contributory patent infringement, then any patent licenses
+ granted to You under this License for that Work shall terminate
+ as of the date such litigation is filed.
+
+ 4. Redistribution. You may reproduce and distribute copies of the
+ Work or Derivative Works thereof in any medium, with or without
+ modifications, and in Source or Object form, provided that You
+ meet the following conditions:
+
+ (a) You must give any other recipients of the Work or
+ Derivative Works a copy of this License; and
+
+ (b) You must cause any modified files to carry prominent notices
+ stating that You changed the files; and
+
+ (c) You must retain, in the Source form of any Derivative Works
+ that You distribute, all copyright, patent, trademark, and
+ attribution notices from the Source form of the Work,
+ excluding those notices that do not pertain to any part of
+ the Derivative Works; and
+
+ (d) If the Work includes a "NOTICE" text file as part of its
+ distribution, then any Derivative Works that You distribute must
+ include a readable copy of the attribution notices contained
+ within such NOTICE file, excluding those notices that do not
+ pertain to any part of the Derivative Works, in at least one
+ of the following places: within a NOTICE text file distributed
+ as part of the Derivative Works; within the Source form or
+ documentation, if provided along with the Derivative Works; or,
+ within a display generated by the Derivative Works, if and
+ wherever such third-party notices normally appear. The contents
+ of the NOTICE file are for informational purposes only and
+ do not modify the License. You may add Your own attribution
+ notices within Derivative Works that You distribute, alongside
+ or as an addendum to the NOTICE text from the Work, provided
+ that such additional attribution notices cannot be construed
+ as modifying the License.
+
+ You may add Your own copyright statement to Your modifications and
+ may provide additional or different license terms and conditions
+ for use, reproduction, or distribution of Your modifications, or
+ for any such Derivative Works as a whole, provided Your use,
+ reproduction, and distribution of the Work otherwise complies with
+ the conditions stated in this License.
+
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
+ any Contribution intentionally submitted for inclusion in the Work
+ by You to the Licensor shall be under the terms and conditions of
+ this License, without any additional terms or conditions.
+ Notwithstanding the above, nothing herein shall supersede or modify
+ the terms of any separate license agreement you may have executed
+ with Licensor regarding such Contributions.
+
+ 6. Trademarks. This License does not grant permission to use the trade
+ names, trademarks, service marks, or product names of the Licensor,
+ except as required for reasonable and customary use in describing the
+ origin of the Work and reproducing the content of the NOTICE file.
+
+ 7. Disclaimer of Warranty. Unless required by applicable law or
+ agreed to in writing, Licensor provides the Work (and each
+ Contributor provides its Contributions) on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied, including, without limitation, any warranties or conditions
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+ PARTICULAR PURPOSE. You are solely responsible for determining the
+ appropriateness of using or redistributing the Work and assume any
+ risks associated with Your exercise of permissions under this License.
+
+ 8. Limitation of Liability. In no event and under no legal theory,
+ whether in tort (including negligence), contract, or otherwise,
+ unless required by applicable law (such as deliberate and grossly
+ negligent acts) or agreed to in writing, shall any Contributor be
+ liable to You for damages, including any direct, indirect, special,
+ incidental, or consequential damages of any character arising as a
+ result of this License or out of the use or inability to use the
+ Work (including but not limited to damages for loss of goodwill,
+ work stoppage, computer failure or malfunction, or any and all
+ other commercial damages or losses), even if such Contributor
+ has been advised of the possibility of such damages.
+
+ 9. Accepting Warranty or Additional Liability. While redistributing
+ the Work or Derivative Works thereof, You may choose to offer,
+ and charge a fee for, acceptance of support, warranty, indemnity,
+ or other liability obligations and/or rights consistent with this
+ License. However, in accepting such obligations, You may act only
+ on Your own behalf and on Your sole responsibility, not on behalf
+ of any other Contributor, and only if You agree to indemnify,
+ defend, and hold each Contributor harmless for any liability
+ incurred by, or claims asserted against, such Contributor by reason
+ of your accepting any such warranty or additional liability.
+
+ END OF TERMS AND CONDITIONS
+
+ APPENDIX: How to apply the Apache License to your work.
+
+ To apply the Apache License to your work, attach the following
+ boilerplate notice, with the fields enclosed by brackets "[]"
+ replaced with your own identifying information. (Don't include
+ the brackets!) The text should be enclosed in the appropriate
+ comment syntax for the file format. We also recommend that a
+ file or class name and description of purpose be included on the
+ same "printed page" as the copyright notice for easier
+ identification within third-party archives.
+
+ Copyright [yyyy] [name of copyright owner]
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+--------------------------------------------------------------------------------
diff --git a/.github/workflows/python-ci.yml b/python/MANIFEST.in
similarity index 55%
copy from .github/workflows/python-ci.yml
copy to python/MANIFEST.in
index 0c5a682..a31dc63 100644
--- a/.github/workflows/python-ci.yml
+++ b/python/MANIFEST.in
@@ -1,4 +1,3 @@
-#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -15,35 +14,5 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-#
-
-name: "Python CI"
-on:
- push:
- paths:
- - '.github/workflows/python-ci.yml'
- - 'python_legacy/**'
- pull_request:
- paths:
- - '.github/workflows/python-ci.yml'
- - 'python_legacy/**'
-
-jobs:
- tox:
- runs-on: ubuntu-latest
- strategy:
- matrix:
- python: [3.7, 3.8, 3.9]
-
- steps:
- - uses: actions/checkout@v2
- - uses: actions/setup-python@v2
- with:
- python-version: ${{ matrix.python }}
- - working-directory: ./python_legacy
- run: |
- pip install -e .[dev]
- pip install -U tox-gh-actions
- - working-directory: ./python_legacy
- run: tox
+graft src
diff --git a/python/NOTICE b/python/NOTICE
new file mode 100644
index 0000000..906eca3c0
--- /dev/null
+++ b/python/NOTICE
@@ -0,0 +1,8 @@
+
+Apache Iceberg
+Copyright 2017-2021 The Apache Software Foundation
+
+This product includes software developed at
+The Apache Software Foundation (http://www.apache.org/).
+
+--------------------------------------------------------------------------------
diff --git a/python/README.md b/python/README.md
new file mode 100644
index 0000000..35fe612
--- /dev/null
+++ b/python/README.md
@@ -0,0 +1,43 @@
+<!--
+ - Licensed to the Apache Software Foundation (ASF) under one or more
+ - contributor license agreements. See the NOTICE file distributed with
+ - this work for additional information regarding copyright ownership.
+ - The ASF licenses this file to You under the Apache License, Version 2.0
+ - (the "License"); you may not use this file except in compliance with
+ - the License. You may obtain a copy of the License at
+ -
+ - http://www.apache.org/licenses/LICENSE-2.0
+ -
+ - Unless required by applicable law or agreed to in writing, software
+ - distributed under the License is distributed on an "AS IS" BASIS,
+ - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ - See the License for the specific language governing permissions and
+ - limitations under the License.
+ -->
+
+# Iceberg Python
+
+py-iceberg is a python library for programmatic access to iceberg table metadata as well as to table data in iceberg format.
+It is an implementation of [iceberg table spec](https://iceberg.apache.org/spec/) in Python.
+
+## Getting Started
+
+py-iceberg is currently in development, for development and testing purposes the best way to install the library is to perform the following steps:
+
+```
+git clone https://github.com/apache/iceberg.git
+cd iceberg/python
+pip install -e .
+```
+
+## Testing
+
+Testing is done using tox. The config can be found in `tox.ini` within the python directory of the iceberg project.
+
+```
+# simply run tox within the python dir
+tox
+```
+
+## Get in Touch
+- [Iceberg community](https://iceberg.apache.org/community/)
diff --git a/.github/workflows/python-ci.yml b/python/pyproject.toml
similarity index 55%
copy from .github/workflows/python-ci.yml
copy to python/pyproject.toml
index 0c5a682..7389f49 100644
--- a/.github/workflows/python-ci.yml
+++ b/python/pyproject.toml
@@ -1,4 +1,3 @@
-#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -15,35 +14,10 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-#
-
-name: "Python CI"
-on:
- push:
- paths:
- - '.github/workflows/python-ci.yml'
- - 'python_legacy/**'
- pull_request:
- paths:
- - '.github/workflows/python-ci.yml'
- - 'python_legacy/**'
-
-jobs:
- tox:
- runs-on: ubuntu-latest
- strategy:
- matrix:
- python: [3.7, 3.8, 3.9]
-
- steps:
- - uses: actions/checkout@v2
- - uses: actions/setup-python@v2
- with:
- python-version: ${{ matrix.python }}
- - working-directory: ./python_legacy
- run: |
- pip install -e .[dev]
- pip install -U tox-gh-actions
- - working-directory: ./python_legacy
- run: tox
+[build-system]
+requires = [
+ "setuptools>=42",
+ "wheel"
+]
+build-backend = "setuptools.build_meta"
\ No newline at end of file
diff --git a/python/setup.cfg b/python/setup.cfg
new file mode 100644
index 0000000..5c58314
--- /dev/null
+++ b/python/setup.cfg
@@ -0,0 +1,47 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+[metadata]
+name = py-iceberg
+version = 0.1.0a1
+author = Apache Software Foundation
+author_email = dev@iceberg.apache.org
+description = Apache Iceberg is an open table format for huge analytic datasets.
+long_description = file: README.md
+long_description_content_type = text/markdown
+url = https://iceberg.apache.org/
+project_urls =
+ Source Code = https://github.com/apache/iceberg/
+ Issue Tracker = https://github.com/apache/iceberg/issues
+license="Apache License 2.0",
+license_files =
+ LICENSE
+classifiers =
+ License :: OSI Approved :: Apache Software License
+ Operating System :: OS Independent
+ Programming Language :: Python :: 3.7
+ Programming Language :: Python :: 3.8
+ Programming Language :: Python :: 3.9
+
+[options]
+package_dir =
+ = src
+packages = find:
+python_requires = >=3.7
+
+[options.packages.find]
+where = src
diff --git a/.github/workflows/python-ci.yml b/python/setup.py
similarity index 56%
copy from .github/workflows/python-ci.yml
copy to python/setup.py
index 0c5a682..9acba2c 100644
--- a/.github/workflows/python-ci.yml
+++ b/python/setup.py
@@ -1,4 +1,3 @@
-#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -15,35 +14,16 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-#
-
-name: "Python CI"
-on:
- push:
- paths:
- - '.github/workflows/python-ci.yml'
- - 'python_legacy/**'
- pull_request:
- paths:
- - '.github/workflows/python-ci.yml'
- - 'python_legacy/**'
-
-jobs:
- tox:
- runs-on: ubuntu-latest
- strategy:
- matrix:
- python: [3.7, 3.8, 3.9]
- steps:
- - uses: actions/checkout@v2
- - uses: actions/setup-python@v2
- with:
- python-version: ${{ matrix.python }}
- - working-directory: ./python_legacy
- run: |
- pip install -e .[dev]
- pip install -U tox-gh-actions
- - working-directory: ./python_legacy
- run: tox
+from setuptools import setup
+setup(
+ name='py-iceberg',
+ install_requires=[],
+ extras_require={
+ "dev": [
+ "tox-travis==0.12",
+ "pytest",
+ ],
+ }
+)
diff --git a/.github/workflows/python-ci.yml b/python/src/iceberg/__init__.py
similarity index 55%
copy from .github/workflows/python-ci.yml
copy to python/src/iceberg/__init__.py
index 0c5a682..13a8339 100644
--- a/.github/workflows/python-ci.yml
+++ b/python/src/iceberg/__init__.py
@@ -1,4 +1,3 @@
-#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -15,35 +14,3 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-#
-
-name: "Python CI"
-on:
- push:
- paths:
- - '.github/workflows/python-ci.yml'
- - 'python_legacy/**'
- pull_request:
- paths:
- - '.github/workflows/python-ci.yml'
- - 'python_legacy/**'
-
-jobs:
- tox:
- runs-on: ubuntu-latest
- strategy:
- matrix:
- python: [3.7, 3.8, 3.9]
-
- steps:
- - uses: actions/checkout@v2
- - uses: actions/setup-python@v2
- with:
- python-version: ${{ matrix.python }}
- - working-directory: ./python_legacy
- run: |
- pip install -e .[dev]
- pip install -U tox-gh-actions
- - working-directory: ./python_legacy
- run: tox
-
diff --git a/.github/workflows/python-ci.yml b/python/src/iceberg/catalog/__init__.py
similarity index 55%
copy from .github/workflows/python-ci.yml
copy to python/src/iceberg/catalog/__init__.py
index 0c5a682..13a8339 100644
--- a/.github/workflows/python-ci.yml
+++ b/python/src/iceberg/catalog/__init__.py
@@ -1,4 +1,3 @@
-#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -15,35 +14,3 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-#
-
-name: "Python CI"
-on:
- push:
- paths:
- - '.github/workflows/python-ci.yml'
- - 'python_legacy/**'
- pull_request:
- paths:
- - '.github/workflows/python-ci.yml'
- - 'python_legacy/**'
-
-jobs:
- tox:
- runs-on: ubuntu-latest
- strategy:
- matrix:
- python: [3.7, 3.8, 3.9]
-
- steps:
- - uses: actions/checkout@v2
- - uses: actions/setup-python@v2
- with:
- python-version: ${{ matrix.python }}
- - working-directory: ./python_legacy
- run: |
- pip install -e .[dev]
- pip install -U tox-gh-actions
- - working-directory: ./python_legacy
- run: tox
-
diff --git a/.github/workflows/python-ci.yml b/python/src/iceberg/io/__init__.py
similarity index 55%
copy from .github/workflows/python-ci.yml
copy to python/src/iceberg/io/__init__.py
index 0c5a682..13a8339 100644
--- a/.github/workflows/python-ci.yml
+++ b/python/src/iceberg/io/__init__.py
@@ -1,4 +1,3 @@
-#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -15,35 +14,3 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-#
-
-name: "Python CI"
-on:
- push:
- paths:
- - '.github/workflows/python-ci.yml'
- - 'python_legacy/**'
- pull_request:
- paths:
- - '.github/workflows/python-ci.yml'
- - 'python_legacy/**'
-
-jobs:
- tox:
- runs-on: ubuntu-latest
- strategy:
- matrix:
- python: [3.7, 3.8, 3.9]
-
- steps:
- - uses: actions/checkout@v2
- - uses: actions/setup-python@v2
- with:
- python-version: ${{ matrix.python }}
- - working-directory: ./python_legacy
- run: |
- pip install -e .[dev]
- pip install -U tox-gh-actions
- - working-directory: ./python_legacy
- run: tox
-
diff --git a/.github/workflows/python-ci.yml b/python/src/iceberg/table/__init__.py
similarity index 55%
copy from .github/workflows/python-ci.yml
copy to python/src/iceberg/table/__init__.py
index 0c5a682..13a8339 100644
--- a/.github/workflows/python-ci.yml
+++ b/python/src/iceberg/table/__init__.py
@@ -1,4 +1,3 @@
-#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -15,35 +14,3 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-#
-
-name: "Python CI"
-on:
- push:
- paths:
- - '.github/workflows/python-ci.yml'
- - 'python_legacy/**'
- pull_request:
- paths:
- - '.github/workflows/python-ci.yml'
- - 'python_legacy/**'
-
-jobs:
- tox:
- runs-on: ubuntu-latest
- strategy:
- matrix:
- python: [3.7, 3.8, 3.9]
-
- steps:
- - uses: actions/checkout@v2
- - uses: actions/setup-python@v2
- with:
- python-version: ${{ matrix.python }}
- - working-directory: ./python_legacy
- run: |
- pip install -e .[dev]
- pip install -U tox-gh-actions
- - working-directory: ./python_legacy
- run: tox
-
diff --git a/.github/workflows/python-ci.yml b/python/src/iceberg/utils/__init__.py
similarity index 55%
copy from .github/workflows/python-ci.yml
copy to python/src/iceberg/utils/__init__.py
index 0c5a682..13a8339 100644
--- a/.github/workflows/python-ci.yml
+++ b/python/src/iceberg/utils/__init__.py
@@ -1,4 +1,3 @@
-#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -15,35 +14,3 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-#
-
-name: "Python CI"
-on:
- push:
- paths:
- - '.github/workflows/python-ci.yml'
- - 'python_legacy/**'
- pull_request:
- paths:
- - '.github/workflows/python-ci.yml'
- - 'python_legacy/**'
-
-jobs:
- tox:
- runs-on: ubuntu-latest
- strategy:
- matrix:
- python: [3.7, 3.8, 3.9]
-
- steps:
- - uses: actions/checkout@v2
- - uses: actions/setup-python@v2
- with:
- python-version: ${{ matrix.python }}
- - working-directory: ./python_legacy
- run: |
- pip install -e .[dev]
- pip install -U tox-gh-actions
- - working-directory: ./python_legacy
- run: tox
-
diff --git a/python/src/iceberg/utils/bin_packing.py b/python/src/iceberg/utils/bin_packing.py
new file mode 100644
index 0000000..17d4faa
--- /dev/null
+++ b/python/src/iceberg/utils/bin_packing.py
@@ -0,0 +1,81 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+class PackingIterator:
+ def __init__(self, items, target_weight, lookback, weight_func, largest_bin_first=False):
+ self.items = iter(items)
+ self.target_weight = target_weight
+ self.lookback = lookback
+ self.weight_func = weight_func
+ self.largest_bin_first = largest_bin_first
+ self.bins = list()
+
+ def __iter__(self):
+ return self
+
+ def __next__(self):
+ while True:
+ try:
+ item = next(self.items)
+ weight = self.weight_func(item)
+ bin_ = self.find_bin(weight)
+ if bin_ is not None:
+ bin_.add(item, weight)
+ else:
+ bin_ = self.Bin(self.target_weight)
+ bin_.add(item, weight)
+ self.bins.append(bin_)
+
+ if len(self.bins) > self.lookback:
+ return list(self.remove_bin().items)
+ except StopIteration:
+ break
+
+ if len(self.bins) == 0:
+ raise StopIteration()
+
+ return list(self.remove_bin().items)
+
+ def find_bin(self, weight):
+ for bin_ in self.bins:
+ if bin_.can_add(weight):
+ return bin_
+ return None
+
+ def remove_bin(self):
+ if self.largest_bin_first:
+ bin_ = max(self.bins, key=lambda b: b.weight())
+ self.bins.remove(bin_)
+ return bin_
+ else:
+ return self.bins.pop(0)
+
+ class Bin:
+ def __init__(self, target_weight: int):
+ self.bin_weight = 0
+ self.target_weight = target_weight
+ self.items: list = list()
+
+ def weight(self) -> int:
+ return self.bin_weight
+
+ def can_add(self, weight) -> bool:
+ return self.bin_weight + weight <= self.target_weight
+
+ def add(self, item, weight):
+ self.bin_weight += weight
+ self.items.append(item)
diff --git a/.github/workflows/python-ci.yml b/python/tests/__init__.py
similarity index 55%
copy from .github/workflows/python-ci.yml
copy to python/tests/__init__.py
index 0c5a682..13a8339 100644
--- a/.github/workflows/python-ci.yml
+++ b/python/tests/__init__.py
@@ -1,4 +1,3 @@
-#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -15,35 +14,3 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-#
-
-name: "Python CI"
-on:
- push:
- paths:
- - '.github/workflows/python-ci.yml'
- - 'python_legacy/**'
- pull_request:
- paths:
- - '.github/workflows/python-ci.yml'
- - 'python_legacy/**'
-
-jobs:
- tox:
- runs-on: ubuntu-latest
- strategy:
- matrix:
- python: [3.7, 3.8, 3.9]
-
- steps:
- - uses: actions/checkout@v2
- - uses: actions/setup-python@v2
- with:
- python-version: ${{ matrix.python }}
- - working-directory: ./python_legacy
- run: |
- pip install -e .[dev]
- pip install -U tox-gh-actions
- - working-directory: ./python_legacy
- run: tox
-
diff --git a/.github/workflows/python-ci.yml b/python/tests/utils/__init__.py
similarity index 55%
copy from .github/workflows/python-ci.yml
copy to python/tests/utils/__init__.py
index 0c5a682..13a8339 100644
--- a/.github/workflows/python-ci.yml
+++ b/python/tests/utils/__init__.py
@@ -1,4 +1,3 @@
-#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -15,35 +14,3 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-#
-
-name: "Python CI"
-on:
- push:
- paths:
- - '.github/workflows/python-ci.yml'
- - 'python_legacy/**'
- pull_request:
- paths:
- - '.github/workflows/python-ci.yml'
- - 'python_legacy/**'
-
-jobs:
- tox:
- runs-on: ubuntu-latest
- strategy:
- matrix:
- python: [3.7, 3.8, 3.9]
-
- steps:
- - uses: actions/checkout@v2
- - uses: actions/setup-python@v2
- with:
- python-version: ${{ matrix.python }}
- - working-directory: ./python_legacy
- run: |
- pip install -e .[dev]
- pip install -U tox-gh-actions
- - working-directory: ./python_legacy
- run: tox
-
diff --git a/python/tests/utils/test_bin_packing.py b/python/tests/utils/test_bin_packing.py
new file mode 100644
index 0000000..0cf44f1
--- /dev/null
+++ b/python/tests/utils/test_bin_packing.py
@@ -0,0 +1,50 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import random
+
+from iceberg.utils.bin_packing import PackingIterator
+import pytest
+
+
+@pytest.mark.parametrize("splits, lookback, split_size, open_cost", [
+ ([random.randint(0, 64) for x in range(200)], 20, 128, 4), # random splits
+ ([], 20, 128, 4), # no splits
+ ([0] * 100 + [random.randint(0, 64) in range(10)] + [0] * 100, 20, 128, 4) # sparse
+])
+def test_bin_packing(splits, lookback, split_size, open_cost):
+
+ def weight_func(x):
+ return max(x, open_cost)
+
+ item_list_sums = [sum(item)
+ for item in PackingIterator(splits, split_size, lookback, weight_func)]
+ assert all([split_size >= item_sum >= 0 for item_sum in item_list_sums])
+
+
+@pytest.mark.parametrize("splits, target_weight, lookback, largest_bin_first, expected_lists", [
+ ([36, 36, 36, 36, 73, 110, 128], 128, 2, True, [[110], [128], [36, 73], [36, 36, 36]]),
+ ([36, 36, 36, 36, 73, 110, 128], 128, 2, False, [[36, 36, 36], [36, 73], [110], [128]]),
+ ([64, 64, 128, 32, 32, 32, 32], 128, 1, True, [[64, 64], [128], [32, 32, 32, 32]]),
+ ([64, 64, 128, 32, 32, 32, 32], 128, 1, False, [[64, 64], [128], [32, 32, 32, 32]]),
+])
+def test_bin_packing_lookback(splits, target_weight, lookback, largest_bin_first, expected_lists):
+ def weight_func(x):
+ return x
+
+ assert [item for item in PackingIterator(
+ splits, target_weight, lookback, weight_func, largest_bin_first)] == expected_lists
diff --git a/python/tox.ini b/python/tox.ini
new file mode 100644
index 0000000..d1596ea
--- /dev/null
+++ b/python/tox.ini
@@ -0,0 +1,118 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+[tox]
+envlist = py37,py38,py39,linters
+
+[testenv]
+usedevelop = true
+deps =
+ coverage
+ mock
+ nose
+ pytest
+setenv =
+ COVERAGE_FILE = test-reports/{envname}/.coverage
+ PYTEST_ADDOPTS = --junitxml=test-reports/{envname}/junit.xml -vv
+commands =
+ coverage run --source src --parallel-mode -m pytest {posargs}
+ coverage combine
+ coverage report -m --fail-under=90
+ coverage html -d test-reports/{envname}/coverage-html
+ coverage xml -o test-reports/{envname}/coverage.xml
+
+[testenv:linters]
+basepython = python3
+skip_install = true
+deps =
+ .
+ {[testenv:flake8]deps}
+ {[testenv:bandit]deps}
+ {[testenv:mypy]deps}
+commands =
+ {[testenv:flake8]commands}
+ {[testenv:bandit]commands}
+ {[testenv:mypy]commands}
+
+[testenv:flake8]
+basepython = python3
+skip_install = true
+deps =
+ flake8>=3.8.4
+ flake8-import-order>=0.9
+ flake8-bugbear
+commands =
+ flake8 src setup.py tests
+
+[testenv:mypy]
+basepython = python3
+skip_install = true
+deps =
+ mypy
+ types-pytz
+ types-python-dateutil
+commands =
+ mypy --ignore-missing-imports src/
+
+[testenv:bandit]
+basepython = python3
+skip_install = true
+deps =
+ bandit
+commands =
+ bandit --ini tox.ini -r src
+
+[testenv:docs]
+basepython = python3
+deps =
+ -r docs/source/requirements.txt
+commands =
+ sphinx-build -E -W -c docs/source/ -b html docs/source/ docs/build/html
+ sphinx-build -E -W -c docs/source/ -b man docs/source/ docs/build/man
+
+[testenv:serve-docs]
+basepython = python3
+skip_install = true
+changedir = docs/build/html
+deps =
+commands =
+ python -m http.server {posargs}
+
+[flake8]
+ignore = E501,W503
+exclude =
+ *.egg-info,
+ *.pyc,
+ .cache,
+ .coverage.*,
+ .gradle,
+ .tox,
+ build,
+ dist,
+ htmlcov.*
+max-complexity = 10
+import-order-style = google
+application-import-names = flake8
+
+[pytest]
+norecursedirs=.*
+
+[gh-actions]
+python =
+ 3.7: py37
+ 3.8: py38, linters
+ 3.9: py39