You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by jo...@apache.org on 2021/06/19 05:41:19 UTC
[arrow-experimental-rs-parquet2] branch master updated: Removed all.
This is an automated email from the ASF dual-hosted git repository.
jorgecarleitao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow-experimental-rs-parquet2.git
The following commit(s) were added to refs/heads/master by this push:
new 5518118 Removed all.
5518118 is described below
commit 55181187208b147ca0efa20beb7c808a3168c1cc
Author: Jorge C. Leitao <jo...@gmail.com>
AuthorDate: Sat Jun 19 05:40:27 2021 +0000
Removed all.
This will allow to start a project from scratch without losing
the contributions' stats and others.
---
.gitattributes | 6 -
.github/ISSUE_TEMPLATE/bug_report.md | 20 -
.github/ISSUE_TEMPLATE/feature_request.md | 21 -
.github/pull_request_template.md | 31 -
.github/workflows/cancel.yml | 53 -
.github/workflows/comment_bot.yml | 72 -
.github/workflows/dev.yml | 67 -
.github/workflows/rust.yml | 232 --
.github_changelog_generator | 29 -
.gitignore | 8 -
.gitmodules | 6 -
.pre-commit-config.yaml | 69 -
CHANGELOG.md | 78 -
CODE_OF_CONDUCT.md | 24 -
CONTRIBUTING.md | 77 -
Cargo.toml | 23 -
LICENSE.txt | 2220 -----------------
README.md | 24 -
dev/.gitignore | 22 -
dev/README.md | 57 -
dev/release/README.md | 266 ---
dev/release/check-rat-report.py | 59 -
dev/release/cherry-pick-pr.py | 154 --
dev/release/create-tarball.sh | 123 -
dev/release/rat_exclude_files.txt | 27 -
dev/release/release-tarball.sh | 72 -
dev/release/run-rat.sh | 43 -
dev/release/update_change_log.sh | 39 -
dev/release/verify-release-candidate.sh | 153 --
header | 16 -
parquet-testing | 1 -
parquet/Cargo.toml | 82 -
parquet/README.md | 143 --
parquet/benches/arrow_array_reader.rs | 766 ------
parquet/benches/arrow_writer.rs | 202 --
parquet/build.rs | 43 -
parquet/src/arrow/array_reader.rs | 2556 --------------------
parquet/src/arrow/arrow_array_reader.rs | 1562 ------------
parquet/src/arrow/arrow_reader.rs | 671 ------
parquet/src/arrow/arrow_writer.rs | 1678 -------------
parquet/src/arrow/converter.rs | 454 ----
parquet/src/arrow/levels.rs | 1548 ------------
parquet/src/arrow/mod.rs | 70 -
parquet/src/arrow/record_reader.rs | 802 -------
parquet/src/arrow/schema.rs | 1943 ---------------
parquet/src/basic.rs | 1969 ----------------
parquet/src/bin/parquet-read.rs | 128 -
parquet/src/bin/parquet-rowcount.rs | 87 -
parquet/src/bin/parquet-schema.rs | 104 -
parquet/src/column/mod.rs | 124 -
parquet/src/column/page.rs | 307 ---
parquet/src/column/reader.rs | 1364 -----------
parquet/src/column/writer.rs | 2020 ----------------
parquet/src/compression.rs | 393 ----
parquet/src/data_type.rs | 1365 -----------
parquet/src/encodings/decoding.rs | 1386 -----------
parquet/src/encodings/encoding.rs | 1334 -----------
parquet/src/encodings/levels.rs | 563 -----
parquet/src/encodings/mod.rs | 21 -
parquet/src/encodings/rle.rs | 831 -------
parquet/src/errors.rs | 146 --
parquet/src/file/footer.rs | 263 ---
parquet/src/file/metadata.rs | 789 -------
parquet/src/file/mod.rs | 110 -
parquet/src/file/properties.rs | 680 ------
parquet/src/file/reader.rs | 206 --
parquet/src/file/serialized_reader.rs | 779 ------
parquet/src/file/statistics.rs | 664 ------
parquet/src/file/writer.rs | 1164 ---------
parquet/src/lib.rs | 57 -
parquet/src/record/api.rs | 1846 ---------------
parquet/src/record/mod.rs | 28 -
parquet/src/record/reader.rs | 1667 -------------
parquet/src/record/record_writer.rs | 26 -
parquet/src/record/triplet.rs | 561 -----
parquet/src/schema/mod.rs | 67 -
parquet/src/schema/parser.rs | 1241 ----------
parquet/src/schema/printer.rs | 827 -------
parquet/src/schema/types.rs | 2080 ----------------
parquet/src/schema/visitor.rs | 240 --
parquet/src/util/bit_packing.rs | 3662 -----------------------------
parquet/src/util/bit_util.rs | 1150 ---------
parquet/src/util/cursor.rs | 261 --
parquet/src/util/hash_util.rs | 172 --
parquet/src/util/io.rs | 329 ---
parquet/src/util/memory.rs | 545 -----
parquet/src/util/mod.rs | 28 -
parquet/src/util/test_common/file_util.rs | 73 -
parquet/src/util/test_common/mod.rs | 33 -
parquet/src/util/test_common/page_util.rs | 320 ---
parquet/src/util/test_common/rand_gen.rs | 139 --
parquet/tests/boolean_writer.rs | 100 -
parquet/tests/custom_writer.rs | 100 -
parquet_derive/Cargo.toml | 42 -
parquet_derive/README.md | 98 -
parquet_derive/src/lib.rs | 126 -
parquet_derive/src/parquet_field.rs | 920 --------
parquet_derive_test/Cargo.toml | 32 -
parquet_derive_test/src/lib.rs | 131 --
pre-commit.sh | 88 -
testing | 1 -
101 files changed, 50399 deletions(-)
diff --git a/.gitattributes b/.gitattributes
deleted file mode 100644
index fac7bf8..0000000
--- a/.gitattributes
+++ /dev/null
@@ -1,6 +0,0 @@
-r/R/RcppExports.R linguist-generated=true
-r/R/arrowExports.R linguist-generated=true
-r/src/RcppExports.cpp linguist-generated=true
-r/src/arrowExports.cpp linguist-generated=true
-r/man/*.Rd linguist-generated=true
-
diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md
deleted file mode 100644
index 5600dab..0000000
--- a/.github/ISSUE_TEMPLATE/bug_report.md
+++ /dev/null
@@ -1,20 +0,0 @@
----
-name: Bug report
-about: Create a report to help us improve
-title: ''
-labels: bug
-assignees: ''
-
----
-
-**Describe the bug**
-A clear and concise description of what the bug is.
-
-**To Reproduce**
-Steps to reproduce the behavior:
-
-**Expected behavior**
-A clear and concise description of what you expected to happen.
-
-**Additional context**
-Add any other context about the problem here.
diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md
deleted file mode 100644
index d9883dd..0000000
--- a/.github/ISSUE_TEMPLATE/feature_request.md
+++ /dev/null
@@ -1,21 +0,0 @@
----
-name: Feature request
-about: Suggest an idea for this project
-title: ''
-labels: enhancement
-assignees: ''
-
----
-
-**Is your feature request related to a problem or challenge? Please describe what you are trying to do.**
-A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
-(This section helps Arrow developers understand the context and *why* for this feature, in addition to the *what*)
-
-**Describe the solution you'd like**
-A clear and concise description of what you want to happen.
-
-**Describe alternatives you've considered**
-A clear and concise description of any alternative solutions or features you've considered.
-
-**Additional context**
-Add any other context or screenshots about the feature request here.
diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md
deleted file mode 100644
index db170e3..0000000
--- a/.github/pull_request_template.md
+++ /dev/null
@@ -1,31 +0,0 @@
-# Which issue does this PR close?
-
-<!---
-We generally require a GitHub issue to be filed for all bug fixes and enhancements and this helps us generate change logs for our releases. You can link an issue to this PR using the GitHub syntax. For example `Closes #123` indicates that this PR will close issue #123.
--->
-
-Closes #.
-
-# Rationale for this change
-
- <!---
- Why are you proposing this change? If this is already explained clearly in the issue then this section is not needed.
- Explaining clearly why changes are proposed helps reviewers understand your changes and offer better suggestions for fixes.
--->
-
-# What changes are included in this PR?
-
-<!---
-There is no need to duplicate the description in the issue here but it is sometimes worth providing a summary of the individual changes in this PR.
--->
-
-# Are there any user-facing changes?
-
-
-<!---
-If there are user-facing changes then we may require documentation to be updated before approving the PR.
--->
-
-<!---
-If there are any breaking changes to public APIs, please add the `breaking change` label.
--->
diff --git a/.github/workflows/cancel.yml b/.github/workflows/cancel.yml
deleted file mode 100644
index b4fb904..0000000
--- a/.github/workflows/cancel.yml
+++ /dev/null
@@ -1,53 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-name: Cancel stale runs
-
-on:
- workflow_run:
- # The name of another workflow (whichever one) that always runs on PRs
- workflows: ['Dev']
- types: ['requested']
-
-jobs:
- cancel-stale-workflow-runs:
- name: "Cancel stale workflow runs"
- runs-on: ubuntu-latest
- steps:
- # Unfortunately, we need to define a separate cancellation step for
- # each workflow where we want to cancel stale runs.
- - uses: potiuk/cancel-workflow-runs@master
- name: "Cancel stale Dev runs"
- with:
- cancelMode: allDuplicates
- token: ${{ secrets.GITHUB_TOKEN }}
- workflowFileName: dev.yml
- skipEventTypes: '["push", "schedule"]'
- - uses: potiuk/cancel-workflow-runs@master
- name: "Cancel stale Integration runs"
- with:
- cancelMode: allDuplicates
- token: ${{ secrets.GITHUB_TOKEN }}
- workflowFileName: integration.yml
- skipEventTypes: '["push", "schedule"]'
- - uses: potiuk/cancel-workflow-runs@master
- name: "Cancel stale Rust runs"
- with:
- cancelMode: allDuplicates
- token: ${{ secrets.GITHUB_TOKEN }}
- workflowFileName: rust.yml
- skipEventTypes: '["push", "schedule"]'
diff --git a/.github/workflows/comment_bot.yml b/.github/workflows/comment_bot.yml
deleted file mode 100644
index 6ca0953..0000000
--- a/.github/workflows/comment_bot.yml
+++ /dev/null
@@ -1,72 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-name: Comment Bot
-
-on:
- # TODO(kszucs): support pull_request_review_comment
- issue_comment:
- types:
- - created
- - edited
-
-jobs:
- crossbow:
- name: Listen!
- if: startsWith(github.event.comment.body, '@github-actions crossbow')
- runs-on: ubuntu-latest
- steps:
- - name: Checkout Arrow
- uses: actions/checkout@v2
- with:
- repository: apache/arrow
- - name: Set up Python
- uses: actions/setup-python@v2
- with:
- python-version: 3.8
- - name: Install Archery and Crossbow dependencies
- run: pip install -e dev/archery[bot]
- - name: Handle Github comment event
- env:
- ARROW_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- CROSSBOW_GITHUB_TOKEN: ${{ secrets.CROSSBOW_GITHUB_TOKEN }}
- run: |
- archery trigger-bot \
- --event-name ${{ github.event_name }} \
- --event-payload ${{ github.event_path }}
-
- rebase:
- name: "Rebase"
- if: startsWith(github.event.comment.body, '@github-actions rebase')
- runs-on: ubuntu-latest
- steps:
- - uses: actions/checkout@v2
- - uses: r-lib/actions/pr-fetch@master
- with:
- repo-token: ${{ secrets.GITHUB_TOKEN }}
- - name: Rebase on ${{ github.repository }} master
- run: |
- set -ex
- git config user.name "$(git log -1 --pretty=format:%an)"
- git config user.email "$(git log -1 --pretty=format:%ae)"
- git remote add upstream https://github.com/${{ github.repository }}
- git fetch --unshallow upstream master
- git rebase upstream/master
- - uses: r-lib/actions/pr-push@master
- with:
- repo-token: ${{ secrets.GITHUB_TOKEN }}
- args: "--force"
diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml
deleted file mode 100644
index 548f0dd..0000000
--- a/.github/workflows/dev.yml
+++ /dev/null
@@ -1,67 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-name: Dev
-
-on:
- # always trigger
- push:
- pull_request:
-
-env:
- ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }}
- ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }}
-
-jobs:
-
- rat:
- name: Release Audit Tool (RAT)
- runs-on: ubuntu-latest
- steps:
- - name: Checkout Arrow
- uses: actions/checkout@v2
- with:
- repository: apache/arrow
- submodules: true
- fetch-depth: 0
- - name: Checkout Arrow Rust
- uses: actions/checkout@v2
- with:
- path: rust
- fetch-depth: 0
- - name: Setup Python
- uses: actions/setup-python@v1
- with:
- python-version: 3.8
- - name: Setup Archery
- run: pip install -e dev/archery[lint]
- - name: Lint
- run: archery lint --rat
-
- prettier:
- name: Use prettier to check formatting of documents
- runs-on: ubuntu-latest
- steps:
- - uses: actions/checkout@v2
- - uses: actions/setup-node@v2
- with:
- node-version: "14"
- - name: Prettier check
- run: |
- # if you encounter error, try rerun the command below with --write instead of --check
- # and commit the changes
- npx prettier@2.3.0 --check {arrow,arrow-flight,dev,integration-testing,parquet}/**/*.md README.md CODE_OF_CONDUCT.md CONTRIBUTING.md
diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml
deleted file mode 100644
index f5ab81b..0000000
--- a/.github/workflows/rust.yml
+++ /dev/null
@@ -1,232 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-name: Rust
-
-on:
- # always trigger
- push:
- pull_request:
-
-jobs:
-
- # build the library, a compilation step used by multiple steps below
- linux-build-lib:
- name: Build Libraries on AMD64 Rust ${{ matrix.rust }}
- runs-on: ubuntu-latest
- strategy:
- matrix:
- arch: [amd64]
- rust: [stable]
- container:
- image: ${{ matrix.arch }}/rust
- env:
- # Disable full debug symbol generation to speed up CI build and keep memory down
- # "1" means line tables only, which is useful for panic tracebacks.
- RUSTFLAGS: "-C debuginfo=1"
- steps:
- - uses: actions/checkout@v2
- - name: Cache Cargo
- uses: actions/cache@v2
- with:
- # these represent dependencies downloaded by cargo
- # and thus do not depend on the OS, arch nor rust version.
- path: /github/home/.cargo
- key: cargo-cache-
- - name: Cache Rust dependencies
- uses: actions/cache@v2
- with:
- # these represent compiled steps of both dependencies and arrow
- # and thus are specific for a particular OS, arch and rust version.
- path: /github/home/target
- key: ${{ runner.os }}-${{ matrix.arch }}-target-cache-${{ matrix.rust }}-
- - name: Setup Rust toolchain
- run: |
- rustup toolchain install ${{ matrix.rust }}
- rustup default ${{ matrix.rust }}
- rustup component add rustfmt
- - name: Build Workspace
- run: |
- export CARGO_HOME="/github/home/.cargo"
- export CARGO_TARGET_DIR="/github/home/target"
- cargo build
-
- # test the crate
- linux-test:
- name: Test Workspace on AMD64 Rust ${{ matrix.rust }}
- needs: [linux-build-lib]
- runs-on: ubuntu-latest
- strategy:
- matrix:
- arch: [amd64]
- rust: [stable]
- container:
- image: ${{ matrix.arch }}/rust
- env:
- # Disable full debug symbol generation to speed up CI build and keep memory down
- # "1" means line tables only, which is useful for panic tracebacks.
- RUSTFLAGS: "-C debuginfo=1"
- ARROW_TEST_DATA: /__w/arrow-rs/arrow-rs/testing/data
- PARQUET_TEST_DATA: /__w/arrow-rs/arrow-rs/parquet-testing/data
- steps:
- - uses: actions/checkout@v2
- with:
- submodules: true
- - name: Cache Cargo
- uses: actions/cache@v2
- with:
- path: /github/home/.cargo
- # this key equals the ones on `linux-build-lib` for re-use
- key: cargo-cache-
- - name: Cache Rust dependencies
- uses: actions/cache@v2
- with:
- path: /github/home/target
- # this key equals the ones on `linux-build-lib` for re-use
- key: ${{ runner.os }}-${{ matrix.arch }}-target-cache-${{ matrix.rust }}
- - name: Setup Rust toolchain
- run: |
- rustup toolchain install ${{ matrix.rust }}
- rustup default ${{ matrix.rust }}
- rustup component add rustfmt
- - name: Run tests
- run: |
- export CARGO_HOME="/github/home/.cargo"
- export CARGO_TARGET_DIR="/github/home/target"
- # run tests on all workspace members with default feature list
- cargo test
-
- windows-and-macos:
- name: Test on ${{ matrix.os }} Rust ${{ matrix.rust }}
- runs-on: ${{ matrix.os }}
- strategy:
- matrix:
- os: [windows-latest, macos-latest]
- rust: [stable]
- steps:
- - uses: actions/checkout@v2
- with:
- submodules: true
- # TODO: this won't cache anything, which is expensive. Setup this action
- # with a OS-dependent path.
- - name: Setup Rust toolchain
- run: |
- rustup toolchain install ${{ matrix.rust }}
- rustup default ${{ matrix.rust }}
- rustup component add rustfmt
- - name: Run tests
- shell: bash
- run: |
- export ARROW_TEST_DATA=$(pwd)/testing/data
- export PARQUET_TEST_DATA=$(pwd)/parquet-testing/data
- # do not produce debug symbols to keep memory usage down
- export RUSTFLAGS="-C debuginfo=0"
- cargo test
-
- clippy:
- name: Clippy
- needs: [linux-build-lib]
- runs-on: ubuntu-latest
- strategy:
- matrix:
- arch: [amd64]
- rust: [stable]
- container:
- image: ${{ matrix.arch }}/rust
- env:
- # Disable full debug symbol generation to speed up CI build and keep memory down
- # "1" means line tables only, which is useful for panic tracebacks.
- RUSTFLAGS: "-C debuginfo=1"
- steps:
- - uses: actions/checkout@v2
- with:
- submodules: true
- - name: Cache Cargo
- uses: actions/cache@v2
- with:
- path: /github/home/.cargo
- # this key equals the ones on `linux-build-lib` for re-use
- key: cargo-cache-
- - name: Cache Rust dependencies
- uses: actions/cache@v2
- with:
- path: /github/home/target
- # this key equals the ones on `linux-build-lib` for re-use
- key: ${{ runner.os }}-${{ matrix.arch }}-target-cache-${{ matrix.rust }}
- - name: Setup Rust toolchain
- run: |
- rustup toolchain install ${{ matrix.rust }}
- rustup default ${{ matrix.rust }}
- rustup component add rustfmt clippy
- - name: Run clippy
- run: |
- export CARGO_HOME="/github/home/.cargo"
- export CARGO_TARGET_DIR="/github/home/target"
- cargo clippy --all-targets --workspace -- -D warnings -A clippy::redundant_field_names
-
- lint:
- name: Lint
- runs-on: ubuntu-latest
- container:
- image: amd64/rust
- steps:
- - uses: actions/checkout@v2
- - name: Setup toolchain
- run: |
- rustup toolchain install stable
- rustup default stable
- rustup component add rustfmt
- - name: Run
- run: cargo fmt --all -- --check
- coverage:
- name: Coverage
- runs-on: ubuntu-latest
- strategy:
- matrix:
- arch: [amd64]
- rust: [stable]
- steps:
- - uses: actions/checkout@v2
- with:
- submodules: true
- - name: Cache Cargo
- uses: actions/cache@v2
- with:
- path: /home/runner/.cargo
- # this key is not equal because the user is different than on a container (runner vs github)
- key: cargo-coverage-cache-
- - name: Cache Rust dependencies
- uses: actions/cache@v2
- with:
- path: /home/runner/target
- # this key is not equal because coverage uses different compilation flags.
- key: ${{ runner.os }}-${{ matrix.arch }}-target-coverage-cache-${{ matrix.rust }}-
- - name: Run coverage
- run: |
- export CARGO_HOME="/home/runner/.cargo"
- export CARGO_TARGET_DIR="/home/runner/target"
-
- export ARROW_TEST_DATA=$(pwd)/testing/data
- export PARQUET_TEST_DATA=$(pwd)/parquet-testing/data
-
- # 2020-11-15: There is a cargo-tarpaulin regression in 0.17.0
- # see https://github.com/xd009642/tarpaulin/issues/618
- cargo install --version 0.16.0 cargo-tarpaulin
- cargo tarpaulin --out Xml
- - name: Report coverage
- continue-on-error: true
- run: bash <(curl -s https://codecov.io/bash)
diff --git a/.github_changelog_generator b/.github_changelog_generator
deleted file mode 100644
index 52452ed..0000000
--- a/.github_changelog_generator
+++ /dev/null
@@ -1,29 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-# point to the old changelog in apache/arrow
-front-matter=For older versions, see [apache/arrow/CHANGELOG.md](https://github.com/apache/arrow/blob/master/CHANGELOG.md)\n
-# some issues are just documentation
-add-sections={"documentation":{"prefix":"**Documentation updates:**","labels":["documentation"]}}
-# uncomment to not show PRs. TBD if we shown them or not.
-#pull-requests=false
-# so that the component is shown associated with the issue
-issue-line-labels=arrow,parquet,arrow-flight
-exclude-labels=development-process,invalid
diff --git a/.gitignore b/.gitignore
deleted file mode 100644
index e8d9955..0000000
--- a/.gitignore
+++ /dev/null
@@ -1,8 +0,0 @@
-Cargo.lock
-target
-rusty-tags.vi
-.history
-.flatbuffers/
-.idea/
-.vscode
-venv/*
diff --git a/.gitmodules b/.gitmodules
deleted file mode 100644
index 97da927..0000000
--- a/.gitmodules
+++ /dev/null
@@ -1,6 +0,0 @@
-[submodule "testing"]
- path = testing
- url = https://github.com/apache/arrow-testing
-[submodule "parquet-testing"]
- path = parquet-testing
- url = https://github.com/apache/parquet-testing.git
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
deleted file mode 100644
index 5331a53..0000000
--- a/.pre-commit-config.yaml
+++ /dev/null
@@ -1,69 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# To use this, install the python package `pre-commit` and
-# run once `pre-commit install`. This will setup a git pre-commit-hook
-# that is executed on each commit and will report the linting problems.
-# To run all hooks on all files use `pre-commit run -a`
-
-repos:
- - repo: local
- hooks:
- - id: rat
- name: Release Audit Tool
- language: system
- entry: bash -c "git archive HEAD --prefix=apache-arrow/ --output=arrow-src.tar && ./dev/release/run-rat.sh arrow-src.tar"
- always_run: true
- pass_filenames: false
- - id: rustfmt
- name: Rust Format
- language: system
- entry: bash -c "cargo +stable fmt --all -- --check"
- files: ^.*\.rs$
- types:
- - file
- - rust
- - id: cmake-format
- name: CMake Format
- language: python
- entry: python run-cmake-format.py
- types: [cmake]
- additional_dependencies:
- - cmake_format==0.5.2
- - id: hadolint
- name: Docker Format
- language: docker_image
- types:
- - dockerfile
- entry: --entrypoint /bin/hadolint hadolint/hadolint:latest -
- exclude: ^dev/.*$
- - repo: git://github.com/pre-commit/pre-commit-hooks
- sha: v1.2.3
- hooks:
- - id: flake8
- name: Python Format
- files: ^(python|dev|integration)/
- types:
- - file
- - python
- - id: flake8
- name: Cython Format
- files: ^python/
- types:
- - file
- - cython
- args: [--config=python/.flake8.cython]
diff --git a/CHANGELOG.md b/CHANGELOG.md
deleted file mode 100644
index c901a4e..0000000
--- a/CHANGELOG.md
+++ /dev/null
@@ -1,78 +0,0 @@
-For older versions, see [apache/arrow/CHANGELOG.md](https://github.com/apache/arrow/blob/master/CHANGELOG.md)
-
-# Changelog
-
-## [4.0.1](https://github.com/apache/arrow-rs/tree/4.0.1) (2021-05-16)
-
-[Full Changelog](https://github.com/apache/arrow-rs/compare/8707fd2b2d17b17bd3e79be0255a18ffaea6914a...4.0.1)
-
-**Implemented enhancements:**
-
-- Improve performance of bound checking option [\#280](https://github.com/apache/arrow-rs/issues/280) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)]
-- extend compute kernel arity to include nullary functions [\#276](https://github.com/apache/arrow-rs/issues/276)
-- Add support for pretty-printing Decimal numbers [\#230](https://github.com/apache/arrow-rs/issues/230) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)]
-- CSV Reader String Dictionary Support [\#228](https://github.com/apache/arrow-rs/issues/228) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)]
-- Add Builder interface for adding Arrays to record batches [\#210](https://github.com/apache/arrow-rs/issues/210) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)]
-- Support auto-vectorization for min/max [\#209](https://github.com/apache/arrow-rs/issues/209) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)]
-- Support LargeUtf8 in sort kernel [\#25](https://github.com/apache/arrow-rs/issues/25) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)]
-
-**Fixed bugs:**
-
-- Rust 1.52 Clippy error [\#266](https://github.com/apache/arrow-rs/issues/266)
-- NaNs can break parquet statistics [\#255](https://github.com/apache/arrow-rs/issues/255) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)]
-- u64::MAX does not roundtrip through parquet [\#254](https://github.com/apache/arrow-rs/issues/254) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)]
-- Integration tests failing to compile \(flatbuffer\) [\#249](https://github.com/apache/arrow-rs/issues/249) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)]
-- Fix compatibility quirks between arrow and parquet structs [\#245](https://github.com/apache/arrow-rs/issues/245) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)]
-- Unable to write non-null Arrow structs to Parquet [\#244](https://github.com/apache/arrow-rs/issues/244) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)]
-- Dev PR / Process \(pull\_request\) Failing on PRs [\#242](https://github.com/apache/arrow-rs/issues/242)
-- schema: missing field `metadata` when deserialize [\#241](https://github.com/apache/arrow-rs/issues/241) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)]
-- Arrow does not compile due to flatbuffers upgrade [\#238](https://github.com/apache/arrow-rs/issues/238) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)]
-- Sort with limit panics for the limit includes some but not all nulls, for large arrays [\#235](https://github.com/apache/arrow-rs/issues/235) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)]
-- Read list field correctly in \<struct\<list\>\> [\#167](https://github.com/apache/arrow-rs/issues/167) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)]
-
-**Documentation updates:**
-
-- Comment out the instructions in the PR template [\#277](https://github.com/apache/arrow-rs/issues/277)
-- Update links to datafusion and ballista in README.md [\#19](https://github.com/apache/arrow-rs/issues/19)
-- Update "repository" in Cargo.toml [\#12](https://github.com/apache/arrow-rs/issues/12)
-- Improve docs for NullArray, new\_null\_array and new\_empty\_array [\#240](https://github.com/apache/arrow-rs/pull/240) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([alamb](https://github.com/alamb))
-
-**Merged pull requests:**
-
-- Remove old release scripts [\#293](https://github.com/apache/arrow-rs/pull/293) ([alamb](https://github.com/alamb))
-- manually bump development version [\#288](https://github.com/apache/arrow-rs/pull/288) ([nevi-me](https://github.com/nevi-me))
-- Fix subtraction underflow when sorting string arrays with many nulls [\#285](https://github.com/apache/arrow-rs/pull/285) ([medwards](https://github.com/medwards))
-- Speed up bound checking in `take` [\#281](https://github.com/apache/arrow-rs/pull/281) ([Dandandan](https://github.com/Dandandan))
-- Update PR template by commenting out instructions [\#278](https://github.com/apache/arrow-rs/pull/278) ([nevi-me](https://github.com/nevi-me))
-- Added Decimal support to pretty-print display utility \(\#230\) [\#273](https://github.com/apache/arrow-rs/pull/273) ([mgill25](https://github.com/mgill25))
-- Fix null struct and list roundtrip [\#270](https://github.com/apache/arrow-rs/pull/270) ([nevi-me](https://github.com/nevi-me))
-- 1.52 clippy fixes [\#267](https://github.com/apache/arrow-rs/pull/267) ([nevi-me](https://github.com/nevi-me))
-- Fix typo in csv/reader.rs [\#265](https://github.com/apache/arrow-rs/pull/265) ([domoritz](https://github.com/domoritz))
-- Fix empty Schema::metadata deserialization error [\#260](https://github.com/apache/arrow-rs/pull/260) ([hulunbier](https://github.com/hulunbier))
-- update datafusion and ballista doc links [\#259](https://github.com/apache/arrow-rs/pull/259) ([Jimexist](https://github.com/Jimexist))
-- support full u32 and u64 roundtrip through parquet [\#258](https://github.com/apache/arrow-rs/pull/258) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([crepererum](https://github.com/crepererum))
-- fix NaN handling in parquet statistics [\#256](https://github.com/apache/arrow-rs/pull/256) ([crepererum](https://github.com/crepererum))
-- \[MINOR\] Added env to run rust in integration. [\#253](https://github.com/apache/arrow-rs/pull/253) ([jorgecarleitao](https://github.com/jorgecarleitao))
-- \[Minor\] Made integration tests always run. [\#248](https://github.com/apache/arrow-rs/pull/248) ([jorgecarleitao](https://github.com/jorgecarleitao))
-- fix parquet max\_definition for non-null structs [\#246](https://github.com/apache/arrow-rs/pull/246) ([nevi-me](https://github.com/nevi-me))
-- Disabled rebase needed until demonstrate working. [\#243](https://github.com/apache/arrow-rs/pull/243) ([jorgecarleitao](https://github.com/jorgecarleitao))
-- pin flatbuffers to 0.8.4 [\#239](https://github.com/apache/arrow-rs/pull/239) ([ritchie46](https://github.com/ritchie46))
-- sort\_primitive result is capped to the min of limit or values.len [\#236](https://github.com/apache/arrow-rs/pull/236) ([medwards](https://github.com/medwards))
-- Read list field correctly [\#234](https://github.com/apache/arrow-rs/pull/234) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([nevi-me](https://github.com/nevi-me))
-- Fix code examples for RecordBatch::try\_from\_iter [\#231](https://github.com/apache/arrow-rs/pull/231) ([alamb](https://github.com/alamb))
-- Support string dictionaries in csv reader \(\#228\) [\#229](https://github.com/apache/arrow-rs/pull/229) ([tustvold](https://github.com/tustvold))
-- support LargeUtf8 in sort kernel [\#26](https://github.com/apache/arrow-rs/pull/26) ([ritchie46](https://github.com/ritchie46))
-- Removed unused files [\#22](https://github.com/apache/arrow-rs/pull/22) ([jorgecarleitao](https://github.com/jorgecarleitao))
-- ARROW-12504: Buffer::from\_slice\_ref set correct capacity [\#18](https://github.com/apache/arrow-rs/pull/18) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([tustvold](https://github.com/tustvold))
-- Add GitHub templates [\#17](https://github.com/apache/arrow-rs/pull/17) ([andygrove](https://github.com/andygrove))
-- ARROW-12493: Add support for writing dictionary arrays to CSV and JSON [\#16](https://github.com/apache/arrow-rs/pull/16) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([tustvold](https://github.com/tustvold))
-- ARROW-12426: \[Rust\] Fix concatentation of arrow dictionaries [\#15](https://github.com/apache/arrow-rs/pull/15) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([tustvold](https://github.com/tustvold))
-- Update repository and homepage urls [\#14](https://github.com/apache/arrow-rs/pull/14) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] [[arrow-flight](https://github.com/apache/arrow-rs/labels/arrow-flight)] [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([Dandandan](https://github.com/Dandandan))
-- Added rebase-needed bot [\#13](https://github.com/apache/arrow-rs/pull/13) ([jorgecarleitao](https://github.com/jorgecarleitao))
-- Added Integration tests against arrow [\#10](https://github.com/apache/arrow-rs/pull/10) ([jorgecarleitao](https://github.com/jorgecarleitao))
-- ARROW-12343: \[Rust\] Support auto-vectorization for min/max [\#9](https://github.com/apache/arrow-rs/pull/9) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([Dandandan](https://github.com/Dandandan))
-- ARROW-12411: \[Rust\] Create RecordBatches from Iterators [\#7](https://github.com/apache/arrow-rs/pull/7) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([alamb](https://github.com/alamb))
-
-
-
-\* *This Changelog was automatically generated by [github_changelog_generator](https://github.com/github-changelog-generator/github-changelog-generator)*
diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md
deleted file mode 100644
index 9a24b9b..0000000
--- a/CODE_OF_CONDUCT.md
+++ /dev/null
@@ -1,24 +0,0 @@
-<!---
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
--->
-
-# Code of Conduct
-
-- [Code of Conduct for The Apache Software Foundation][1]
-
-[1]: https://www.apache.org/foundation/policies/conduct.html
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
deleted file mode 100644
index 18d6a7b..0000000
--- a/CONTRIBUTING.md
+++ /dev/null
@@ -1,77 +0,0 @@
-<!---
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
--->
-
-# How to contribute to Apache Arrow
-
-## Did you find a bug?
-
-The Arrow project uses JIRA as a bug tracker. To report a bug, you'll have
-to first create an account on the
-[Apache Foundation JIRA](https://issues.apache.org/jira/). The JIRA server
-hosts bugs and issues for multiple Apache projects. The JIRA project name
-for Arrow is "ARROW".
-
-To be assigned to an issue, ask an Arrow JIRA admin to go to
-[Arrow Roles](https://issues.apache.org/jira/plugins/servlet/project-config/ARROW/roles),
-click "Add users to a role," and add you to the "Contributor" role. Most
-committers are authorized to do this; if you're a committer and aren't
-able to load that project admin page, have someone else add you to the
-necessary role.
-
-Before you create a new bug entry, we recommend you first
-[search](https://issues.apache.org/jira/projects/ARROW/issues/ARROW-5140?filter=allopenissues)
-among existing Arrow issues.
-
-When you create a new JIRA entry, please don't forget to fill the "Component"
-field. Arrow has many subcomponents and this helps triaging and filtering
-tremendously. Also, we conventionally prefix the issue title with the component
-name in brackets, such as "[C++] Crash in Array::Frobnicate()", so as to make
-lists more easy to navigate, and we'd be grateful if you did the same.
-
-## Did you write a patch that fixes a bug or brings an improvement?
-
-First create a JIRA entry as described above. Then, submit your changes
-as a GitHub Pull Request. We'll ask you to prefix the pull request title
-with the JIRA issue number and the component name in brackets.
-(for example: "ARROW-2345: [C++] Fix crash in Array::Frobnicate()").
-Respecting this convention makes it easier for us to process the backlog
-of submitted Pull Requests.
-
-### Minor Fixes
-
-Any functionality change should have a JIRA opened. For minor changes that
-affect documentation, you do not need to open up a JIRA. Instead you can
-prefix the title of your PR with "MINOR: " if meets the following guidelines:
-
-- Grammar, usage and spelling fixes that affect no more than 2 files
-- Documentation updates affecting no more than 2 files and not more
- than 500 words.
-
-## Do you want to propose a significant new feature or an important refactoring?
-
-We ask that all discussions about major changes in the codebase happen
-publicly on the [arrow-dev mailing-list](https://mail-archives.apache.org/mod_mbox/arrow-dev/).
-
-## Do you have questions about the source code, the build procedure or the development process?
-
-You can also ask on the mailing-list, see above.
-
-## Further information
-
-Please read our [development documentation](https://arrow.apache.org/docs/developers/contributing.html).
diff --git a/Cargo.toml b/Cargo.toml
deleted file mode 100644
index 8f8f309..0000000
--- a/Cargo.toml
+++ /dev/null
@@ -1,23 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-[workspace]
-members = [
- "parquet",
- "parquet_derive",
- "parquet_derive_test",
-]
diff --git a/LICENSE.txt b/LICENSE.txt
deleted file mode 100644
index 4cec07f..0000000
--- a/LICENSE.txt
+++ /dev/null
@@ -1,2220 +0,0 @@
-
- Apache License
- Version 2.0, January 2004
- http://www.apache.org/licenses/
-
- TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
- 1. Definitions.
-
- "License" shall mean the terms and conditions for use, reproduction,
- and distribution as defined by Sections 1 through 9 of this document.
-
- "Licensor" shall mean the copyright owner or entity authorized by
- the copyright owner that is granting the License.
-
- "Legal Entity" shall mean the union of the acting entity and all
- other entities that control, are controlled by, or are under common
- control with that entity. For the purposes of this definition,
- "control" means (i) the power, direct or indirect, to cause the
- direction or management of such entity, whether by contract or
- otherwise, or (ii) ownership of fifty percent (50%) or more of the
- outstanding shares, or (iii) beneficial ownership of such entity.
-
- "You" (or "Your") shall mean an individual or Legal Entity
- exercising permissions granted by this License.
-
- "Source" form shall mean the preferred form for making modifications,
- including but not limited to software source code, documentation
- source, and configuration files.
-
- "Object" form shall mean any form resulting from mechanical
- transformation or translation of a Source form, including but
- not limited to compiled object code, generated documentation,
- and conversions to other media types.
-
- "Work" shall mean the work of authorship, whether in Source or
- Object form, made available under the License, as indicated by a
- copyright notice that is included in or attached to the work
- (an example is provided in the Appendix below).
-
- "Derivative Works" shall mean any work, whether in Source or Object
- form, that is based on (or derived from) the Work and for which the
- editorial revisions, annotations, elaborations, or other modifications
- represent, as a whole, an original work of authorship. For the purposes
- of this License, Derivative Works shall not include works that remain
- separable from, or merely link (or bind by name) to the interfaces of,
- the Work and Derivative Works thereof.
-
- "Contribution" shall mean any work of authorship, including
- the original version of the Work and any modifications or additions
- to that Work or Derivative Works thereof, that is intentionally
- submitted to Licensor for inclusion in the Work by the copyright owner
- or by an individual or Legal Entity authorized to submit on behalf of
- the copyright owner. For the purposes of this definition, "submitted"
- means any form of electronic, verbal, or written communication sent
- to the Licensor or its representatives, including but not limited to
- communication on electronic mailing lists, source code control systems,
- and issue tracking systems that are managed by, or on behalf of, the
- Licensor for the purpose of discussing and improving the Work, but
- excluding communication that is conspicuously marked or otherwise
- designated in writing by the copyright owner as "Not a Contribution."
-
- "Contributor" shall mean Licensor and any individual or Legal Entity
- on behalf of whom a Contribution has been received by Licensor and
- subsequently incorporated within the Work.
-
- 2. Grant of Copyright License. Subject to the terms and conditions of
- this License, each Contributor hereby grants to You a perpetual,
- worldwide, non-exclusive, no-charge, royalty-free, irrevocable
- copyright license to reproduce, prepare Derivative Works of,
- publicly display, publicly perform, sublicense, and distribute the
- Work and such Derivative Works in Source or Object form.
-
- 3. Grant of Patent License. Subject to the terms and conditions of
- this License, each Contributor hereby grants to You a perpetual,
- worldwide, non-exclusive, no-charge, royalty-free, irrevocable
- (except as stated in this section) patent license to make, have made,
- use, offer to sell, sell, import, and otherwise transfer the Work,
- where such license applies only to those patent claims licensable
- by such Contributor that are necessarily infringed by their
- Contribution(s) alone or by combination of their Contribution(s)
- with the Work to which such Contribution(s) was submitted. If You
- institute patent litigation against any entity (including a
- cross-claim or counterclaim in a lawsuit) alleging that the Work
- or a Contribution incorporated within the Work constitutes direct
- or contributory patent infringement, then any patent licenses
- granted to You under this License for that Work shall terminate
- as of the date such litigation is filed.
-
- 4. Redistribution. You may reproduce and distribute copies of the
- Work or Derivative Works thereof in any medium, with or without
- modifications, and in Source or Object form, provided that You
- meet the following conditions:
-
- (a) You must give any other recipients of the Work or
- Derivative Works a copy of this License; and
-
- (b) You must cause any modified files to carry prominent notices
- stating that You changed the files; and
-
- (c) You must retain, in the Source form of any Derivative Works
- that You distribute, all copyright, patent, trademark, and
- attribution notices from the Source form of the Work,
- excluding those notices that do not pertain to any part of
- the Derivative Works; and
-
- (d) If the Work includes a "NOTICE" text file as part of its
- distribution, then any Derivative Works that You distribute must
- include a readable copy of the attribution notices contained
- within such NOTICE file, excluding those notices that do not
- pertain to any part of the Derivative Works, in at least one
- of the following places: within a NOTICE text file distributed
- as part of the Derivative Works; within the Source form or
- documentation, if provided along with the Derivative Works; or,
- within a display generated by the Derivative Works, if and
- wherever such third-party notices normally appear. The contents
- of the NOTICE file are for informational purposes only and
- do not modify the License. You may add Your own attribution
- notices within Derivative Works that You distribute, alongside
- or as an addendum to the NOTICE text from the Work, provided
- that such additional attribution notices cannot be construed
- as modifying the License.
-
- You may add Your own copyright statement to Your modifications and
- may provide additional or different license terms and conditions
- for use, reproduction, or distribution of Your modifications, or
- for any such Derivative Works as a whole, provided Your use,
- reproduction, and distribution of the Work otherwise complies with
- the conditions stated in this License.
-
- 5. Submission of Contributions. Unless You explicitly state otherwise,
- any Contribution intentionally submitted for inclusion in the Work
- by You to the Licensor shall be under the terms and conditions of
- this License, without any additional terms or conditions.
- Notwithstanding the above, nothing herein shall supersede or modify
- the terms of any separate license agreement you may have executed
- with Licensor regarding such Contributions.
-
- 6. Trademarks. This License does not grant permission to use the trade
- names, trademarks, service marks, or product names of the Licensor,
- except as required for reasonable and customary use in describing the
- origin of the Work and reproducing the content of the NOTICE file.
-
- 7. Disclaimer of Warranty. Unless required by applicable law or
- agreed to in writing, Licensor provides the Work (and each
- Contributor provides its Contributions) on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
- implied, including, without limitation, any warranties or conditions
- of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
- PARTICULAR PURPOSE. You are solely responsible for determining the
- appropriateness of using or redistributing the Work and assume any
- risks associated with Your exercise of permissions under this License.
-
- 8. Limitation of Liability. In no event and under no legal theory,
- whether in tort (including negligence), contract, or otherwise,
- unless required by applicable law (such as deliberate and grossly
- negligent acts) or agreed to in writing, shall any Contributor be
- liable to You for damages, including any direct, indirect, special,
- incidental, or consequential damages of any character arising as a
- result of this License or out of the use or inability to use the
- Work (including but not limited to damages for loss of goodwill,
- work stoppage, computer failure or malfunction, or any and all
- other commercial damages or losses), even if such Contributor
- has been advised of the possibility of such damages.
-
- 9. Accepting Warranty or Additional Liability. While redistributing
- the Work or Derivative Works thereof, You may choose to offer,
- and charge a fee for, acceptance of support, warranty, indemnity,
- or other liability obligations and/or rights consistent with this
- License. However, in accepting such obligations, You may act only
- on Your own behalf and on Your sole responsibility, not on behalf
- of any other Contributor, and only if You agree to indemnify,
- defend, and hold each Contributor harmless for any liability
- incurred by, or claims asserted against, such Contributor by reason
- of your accepting any such warranty or additional liability.
-
- END OF TERMS AND CONDITIONS
-
- APPENDIX: How to apply the Apache License to your work.
-
- To apply the Apache License to your work, attach the following
- boilerplate notice, with the fields enclosed by brackets "[]"
- replaced with your own identifying information. (Don't include
- the brackets!) The text should be enclosed in the appropriate
- comment syntax for the file format. We also recommend that a
- file or class name and description of purpose be included on the
- same "printed page" as the copyright notice for easier
- identification within third-party archives.
-
- Copyright [yyyy] [name of copyright owner]
-
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
-
---------------------------------------------------------------------------------
-
-src/plasma/fling.cc and src/plasma/fling.h: Apache 2.0
-
-Copyright 2013 Sharvil Nanavati
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
---------------------------------------------------------------------------------
-
-src/plasma/thirdparty/ae: Modified / 3-Clause BSD
-
-Copyright (c) 2006-2010, Salvatore Sanfilippo <antirez at gmail dot com>
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-
- * Redistributions of source code must retain the above copyright notice,
- this list of conditions and the following disclaimer.
- * Redistributions in binary form must reproduce the above copyright
- notice, this list of conditions and the following disclaimer in the
- documentation and/or other materials provided with the distribution.
- * Neither the name of Redis nor the names of its contributors may be used
- to endorse or promote products derived from this software without
- specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
-LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-POSSIBILITY OF SUCH DAMAGE.
-
---------------------------------------------------------------------------------
-
-src/plasma/thirdparty/dlmalloc.c: CC0
-
-This is a version (aka dlmalloc) of malloc/free/realloc written by
-Doug Lea and released to the public domain, as explained at
-http://creativecommons.org/publicdomain/zero/1.0/ Send questions,
-comments, complaints, performance data, etc to dl@cs.oswego.edu
-
---------------------------------------------------------------------------------
-
-src/plasma/common.cc (some portions)
-
-Copyright (c) Austin Appleby (aappleby (AT) gmail)
-
-Some portions of this file are derived from code in the MurmurHash project
-
-All code is released to the public domain. For business purposes, Murmurhash is
-under the MIT license.
-
-https://sites.google.com/site/murmurhash/
-
---------------------------------------------------------------------------------
-
-src/arrow/util (some portions): Apache 2.0, and 3-clause BSD
-
-Some portions of this module are derived from code in the Chromium project,
-copyright (c) Google inc and (c) The Chromium Authors and licensed under the
-Apache 2.0 License or the under the 3-clause BSD license:
-
- Copyright (c) 2013 The Chromium Authors. All rights reserved.
-
- Redistribution and use in source and binary forms, with or without
- modification, are permitted provided that the following conditions are
- met:
-
- * Redistributions of source code must retain the above copyright
- notice, this list of conditions and the following disclaimer.
- * Redistributions in binary form must reproduce the above
- copyright notice, this list of conditions and the following disclaimer
- in the documentation and/or other materials provided with the
- distribution.
- * Neither the name of Google Inc. nor the names of its
- contributors may be used to endorse or promote products derived from
- this software without specific prior written permission.
-
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
---------------------------------------------------------------------------------
-
-This project includes code from Daniel Lemire's FrameOfReference project.
-
-https://github.com/lemire/FrameOfReference/blob/6ccaf9e97160f9a3b299e23a8ef739e711ef0c71/src/bpacking.cpp
-
-Copyright: 2013 Daniel Lemire
-Home page: http://lemire.me/en/
-Project page: https://github.com/lemire/FrameOfReference
-License: Apache License Version 2.0 http://www.apache.org/licenses/LICENSE-2.0
-
---------------------------------------------------------------------------------
-
-This project includes code from the TensorFlow project
-
-Copyright 2015 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
---------------------------------------------------------------------------------
-
-This project includes code from the NumPy project.
-
-https://github.com/numpy/numpy/blob/e1f191c46f2eebd6cb892a4bfe14d9dd43a06c4e/numpy/core/src/multiarray/multiarraymodule.c#L2910
-
-https://github.com/numpy/numpy/blob/68fd82271b9ea5a9e50d4e761061dfcca851382a/numpy/core/src/multiarray/datetime.c
-
-Copyright (c) 2005-2017, NumPy Developers.
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are
-met:
-
- * Redistributions of source code must retain the above copyright
- notice, this list of conditions and the following disclaimer.
-
- * Redistributions in binary form must reproduce the above
- copyright notice, this list of conditions and the following
- disclaimer in the documentation and/or other materials provided
- with the distribution.
-
- * Neither the name of the NumPy Developers nor the names of any
- contributors may be used to endorse or promote products derived
- from this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
---------------------------------------------------------------------------------
-
-This project includes code from the Boost project
-
-Boost Software License - Version 1.0 - August 17th, 2003
-
-Permission is hereby granted, free of charge, to any person or organization
-obtaining a copy of the software and accompanying documentation covered by
-this license (the "Software") to use, reproduce, display, distribute,
-execute, and transmit the Software, and to prepare derivative works of the
-Software, and to permit third-parties to whom the Software is furnished to
-do so, all subject to the following:
-
-The copyright notices in the Software and this entire statement, including
-the above license grant, this restriction and the following disclaimer,
-must be included in all copies of the Software, in whole or in part, and
-all derivative works of the Software, unless such copies or derivative
-works are solely in the form of machine-executable object code generated by
-a source language processor.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
-SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
-FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
-ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
-DEALINGS IN THE SOFTWARE.
-
---------------------------------------------------------------------------------
-
-This project includes code from the FlatBuffers project
-
-Copyright 2014 Google Inc.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
---------------------------------------------------------------------------------
-
-This project includes code from the tslib project
-
-Copyright 2015 Microsoft Corporation. All rights reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
---------------------------------------------------------------------------------
-
-This project includes code from the jemalloc project
-
-https://github.com/jemalloc/jemalloc
-
-Copyright (C) 2002-2017 Jason Evans <ja...@canonware.com>.
-All rights reserved.
-Copyright (C) 2007-2012 Mozilla Foundation. All rights reserved.
-Copyright (C) 2009-2017 Facebook, Inc. All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-1. Redistributions of source code must retain the above copyright notice(s),
- this list of conditions and the following disclaimer.
-2. Redistributions in binary form must reproduce the above copyright notice(s),
- this list of conditions and the following disclaimer in the documentation
- and/or other materials provided with the distribution.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) ``AS IS'' AND ANY EXPRESS
-OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
-MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
-EVENT SHALL THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY DIRECT, INDIRECT,
-INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
-PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
-LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
-OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
-ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
---------------------------------------------------------------------------------
-
-This project includes code from the Go project, BSD 3-clause license + PATENTS
-weak patent termination clause
-(https://github.com/golang/go/blob/master/PATENTS).
-
-Copyright (c) 2009 The Go Authors. All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are
-met:
-
- * Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
- * Redistributions in binary form must reproduce the above
-copyright notice, this list of conditions and the following disclaimer
-in the documentation and/or other materials provided with the
-distribution.
- * Neither the name of Google Inc. nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
---------------------------------------------------------------------------------
-
-This project includes code from the hs2client
-
-https://github.com/cloudera/hs2client
-
-Copyright 2016 Cloudera Inc.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
---------------------------------------------------------------------------------
-
-The script ci/scripts/util_wait_for_it.sh has the following license
-
-Copyright (c) 2016 Giles Hall
-
-Permission is hereby granted, free of charge, to any person obtaining a copy of
-this software and associated documentation files (the "Software"), to deal in
-the Software without restriction, including without limitation the rights to
-use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
-of the Software, and to permit persons to whom the Software is furnished to do
-so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.
-
---------------------------------------------------------------------------------
-
-The script r/configure has the following license (MIT)
-
-Copyright (c) 2017, Jeroen Ooms and Jim Hester
-
-Permission is hereby granted, free of charge, to any person obtaining a copy of
-this software and associated documentation files (the "Software"), to deal in
-the Software without restriction, including without limitation the rights to
-use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
-of the Software, and to permit persons to whom the Software is furnished to do
-so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.
-
---------------------------------------------------------------------------------
-
-cpp/src/arrow/util/logging.cc, cpp/src/arrow/util/logging.h and
-cpp/src/arrow/util/logging-test.cc are adapted from
-Ray Project (https://github.com/ray-project/ray) (Apache 2.0).
-
-Copyright (c) 2016 Ray Project (https://github.com/ray-project/ray)
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
---------------------------------------------------------------------------------
-The files cpp/src/arrow/vendored/datetime/date.h, cpp/src/arrow/vendored/datetime/tz.h,
-cpp/src/arrow/vendored/datetime/tz_private.h, cpp/src/arrow/vendored/datetime/ios.h,
-cpp/src/arrow/vendored/datetime/ios.mm,
-cpp/src/arrow/vendored/datetime/tz.cpp are adapted from
-Howard Hinnant's date library (https://github.com/HowardHinnant/date)
-It is licensed under MIT license.
-
-The MIT License (MIT)
-Copyright (c) 2015, 2016, 2017 Howard Hinnant
-Copyright (c) 2016 Adrian Colomitchi
-Copyright (c) 2017 Florian Dang
-Copyright (c) 2017 Paul Thompson
-Copyright (c) 2018 Tomasz Kamiński
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.
-
---------------------------------------------------------------------------------
-
-The file cpp/src/arrow/util/utf8.h includes code adapted from the page
- https://bjoern.hoehrmann.de/utf-8/decoder/dfa/
-with the following license (MIT)
-
-Copyright (c) 2008-2009 Bjoern Hoehrmann <bj...@hoehrmann.de>
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.
-
---------------------------------------------------------------------------------
-
-The file cpp/src/arrow/vendored/string_view.hpp has the following license
-
-Boost Software License - Version 1.0 - August 17th, 2003
-
-Permission is hereby granted, free of charge, to any person or organization
-obtaining a copy of the software and accompanying documentation covered by
-this license (the "Software") to use, reproduce, display, distribute,
-execute, and transmit the Software, and to prepare derivative works of the
-Software, and to permit third-parties to whom the Software is furnished to
-do so, all subject to the following:
-
-The copyright notices in the Software and this entire statement, including
-the above license grant, this restriction and the following disclaimer,
-must be included in all copies of the Software, in whole or in part, and
-all derivative works of the Software, unless such copies or derivative
-works are solely in the form of machine-executable object code generated by
-a source language processor.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
-SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
-FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
-ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
-DEALINGS IN THE SOFTWARE.
-
---------------------------------------------------------------------------------
-
-The files in cpp/src/arrow/vendored/xxhash/ have the following license
-(BSD 2-Clause License)
-
-xxHash Library
-Copyright (c) 2012-2014, Yann Collet
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without modification,
-are permitted provided that the following conditions are met:
-
-* Redistributions of source code must retain the above copyright notice, this
- list of conditions and the following disclaimer.
-
-* Redistributions in binary form must reproduce the above copyright notice, this
- list of conditions and the following disclaimer in the documentation and/or
- other materials provided with the distribution.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
-ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
-ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
-ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-You can contact the author at :
-- xxHash homepage: http://www.xxhash.com
-- xxHash source repository : https://github.com/Cyan4973/xxHash
-
---------------------------------------------------------------------------------
-
-The files in cpp/src/arrow/vendored/double-conversion/ have the following license
-(BSD 3-Clause License)
-
-Copyright 2006-2011, the V8 project authors. All rights reserved.
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are
-met:
-
- * Redistributions of source code must retain the above copyright
- notice, this list of conditions and the following disclaimer.
- * Redistributions in binary form must reproduce the above
- copyright notice, this list of conditions and the following
- disclaimer in the documentation and/or other materials provided
- with the distribution.
- * Neither the name of Google Inc. nor the names of its
- contributors may be used to endorse or promote products derived
- from this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
---------------------------------------------------------------------------------
-
-The files in cpp/src/arrow/vendored/uriparser/ have the following license
-(BSD 3-Clause License)
-
-uriparser - RFC 3986 URI parsing library
-
-Copyright (C) 2007, Weijia Song <so...@gmail.com>
-Copyright (C) 2007, Sebastian Pipping <se...@pipping.org>
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions
-are met:
-
- * Redistributions of source code must retain the above
- copyright notice, this list of conditions and the following
- disclaimer.
-
- * Redistributions in binary form must reproduce the above
- copyright notice, this list of conditions and the following
- disclaimer in the documentation and/or other materials
- provided with the distribution.
-
- * Neither the name of the <ORGANIZATION> nor the names of its
- contributors may be used to endorse or promote products
- derived from this software without specific prior written
- permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
-FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
-COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
-INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
-SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
-HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
-STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
-OF THE POSSIBILITY OF SUCH DAMAGE.
-
---------------------------------------------------------------------------------
-
-The files under dev/tasks/conda-recipes have the following license
-
-BSD 3-clause license
-Copyright (c) 2015-2018, conda-forge
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without modification,
-are permitted provided that the following conditions are met:
-
-1. Redistributions of source code must retain the above copyright notice, this
- list of conditions and the following disclaimer.
-
-2. Redistributions in binary form must reproduce the above copyright notice,
- this list of conditions and the following disclaimer in the documentation
- and/or other materials provided with the distribution.
-
-3. Neither the name of the copyright holder nor the names of its contributors
- may be used to endorse or promote products derived from this software without
- specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
-ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
-FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
-SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
-CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR
-TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
-THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
---------------------------------------------------------------------------------
-
-The files in cpp/src/arrow/vendored/utfcpp/ have the following license
-
-Copyright 2006-2018 Nemanja Trifunovic
-
-Permission is hereby granted, free of charge, to any person or organization
-obtaining a copy of the software and accompanying documentation covered by
-this license (the "Software") to use, reproduce, display, distribute,
-execute, and transmit the Software, and to prepare derivative works of the
-Software, and to permit third-parties to whom the Software is furnished to
-do so, all subject to the following:
-
-The copyright notices in the Software and this entire statement, including
-the above license grant, this restriction and the following disclaimer,
-must be included in all copies of the Software, in whole or in part, and
-all derivative works of the Software, unless such copies or derivative
-works are solely in the form of machine-executable object code generated by
-a source language processor.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
-SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
-FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
-ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
-DEALINGS IN THE SOFTWARE.
-
---------------------------------------------------------------------------------
-
-This project includes code from Apache Kudu.
-
- * cpp/cmake_modules/CompilerInfo.cmake is based on Kudu's cmake_modules/CompilerInfo.cmake
-
-Copyright: 2016 The Apache Software Foundation.
-Home page: https://kudu.apache.org/
-License: http://www.apache.org/licenses/LICENSE-2.0
-
---------------------------------------------------------------------------------
-
-This project includes code from Apache Impala (incubating), formerly
-Impala. The Impala code and rights were donated to the ASF as part of the
-Incubator process after the initial code imports into Apache Parquet.
-
-Copyright: 2012 Cloudera, Inc.
-Copyright: 2016 The Apache Software Foundation.
-Home page: http://impala.apache.org/
-License: http://www.apache.org/licenses/LICENSE-2.0
-
---------------------------------------------------------------------------------
-
-This project includes code from Apache Aurora.
-
-* dev/release/{release,changelog,release-candidate} are based on the scripts from
- Apache Aurora
-
-Copyright: 2016 The Apache Software Foundation.
-Home page: https://aurora.apache.org/
-License: http://www.apache.org/licenses/LICENSE-2.0
-
---------------------------------------------------------------------------------
-
-This project includes code from the Google styleguide.
-
-* cpp/build-support/cpplint.py is based on the scripts from the Google styleguide.
-
-Copyright: 2009 Google Inc. All rights reserved.
-Homepage: https://github.com/google/styleguide
-License: 3-clause BSD
-
---------------------------------------------------------------------------------
-
-This project includes code from Snappy.
-
-* cpp/cmake_modules/{SnappyCMakeLists.txt,SnappyConfig.h} are based on code
- from Google's Snappy project.
-
-Copyright: 2009 Google Inc. All rights reserved.
-Homepage: https://github.com/google/snappy
-License: 3-clause BSD
-
---------------------------------------------------------------------------------
-
-This project includes code from the manylinux project.
-
-* python/manylinux1/scripts/{build_python.sh,python-tag-abi-tag.py,
- requirements.txt} are based on code from the manylinux project.
-
-Copyright: 2016 manylinux
-Homepage: https://github.com/pypa/manylinux
-License: The MIT License (MIT)
-
---------------------------------------------------------------------------------
-
-This project includes code from the cymove project:
-
-* python/pyarrow/includes/common.pxd includes code from the cymove project
-
-The MIT License (MIT)
-Copyright (c) 2019 Omer Ozarslan
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
-IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
-DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
-OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE
-OR OTHER DEALINGS IN THE SOFTWARE.
-
---------------------------------------------------------------------------------
-
-The projects includes code from the Ursabot project under the dev/archery
-directory.
-
-License: BSD 2-Clause
-
-Copyright 2019 RStudio, Inc.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-
-1. Redistributions of source code must retain the above copyright notice, this
- list of conditions and the following disclaimer.
-
-2. Redistributions in binary form must reproduce the above copyright notice,
- this list of conditions and the following disclaimer in the documentation
- and/or other materials provided with the distribution.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
-ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
-FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
-SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
-CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
-OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
---------------------------------------------------------------------------------
-
-This project include code from mingw-w64.
-
-* cpp/src/arrow/util/cpu-info.cc has a polyfill for mingw-w64 < 5
-
-Copyright (c) 2009 - 2013 by the mingw-w64 project
-Homepage: https://mingw-w64.org
-License: Zope Public License (ZPL) Version 2.1.
-
----------------------------------------------------------------------------------
-
-This project include code from Google's Asylo project.
-
-* cpp/src/arrow/result.h is based on status_or.h
-
-Copyright (c) Copyright 2017 Asylo authors
-Homepage: https://asylo.dev/
-License: Apache 2.0
-
---------------------------------------------------------------------------------
-
-This project includes code from Google's protobuf project
-
-* cpp/src/arrow/result.h ARROW_ASSIGN_OR_RAISE is based off ASSIGN_OR_RETURN
-
-Copyright 2008 Google Inc. All rights reserved.
-Homepage: https://developers.google.com/protocol-buffers/
-License:
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are
-met:
-
- * Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
- * Redistributions in binary form must reproduce the above
-copyright notice, this list of conditions and the following disclaimer
-in the documentation and/or other materials provided with the
-distribution.
- * Neither the name of Google Inc. nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-Code generated by the Protocol Buffer compiler is owned by the owner
-of the input file used when generating it. This code is not
-standalone and requires a support library to be linked with it. This
-support library is itself covered by the above license.
-
---------------------------------------------------------------------------------
-
-3rdparty dependency LLVM is statically linked in certain binary distributions.
-Additionally some sections of source code have been derived from sources in LLVM
-and have been clearly labeled as such. LLVM has the following license:
-
-==============================================================================
-The LLVM Project is under the Apache License v2.0 with LLVM Exceptions:
-==============================================================================
-
- Apache License
- Version 2.0, January 2004
- http://www.apache.org/licenses/
-
- TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
- 1. Definitions.
-
- "License" shall mean the terms and conditions for use, reproduction,
- and distribution as defined by Sections 1 through 9 of this document.
-
- "Licensor" shall mean the copyright owner or entity authorized by
- the copyright owner that is granting the License.
-
- "Legal Entity" shall mean the union of the acting entity and all
- other entities that control, are controlled by, or are under common
- control with that entity. For the purposes of this definition,
- "control" means (i) the power, direct or indirect, to cause the
- direction or management of such entity, whether by contract or
- otherwise, or (ii) ownership of fifty percent (50%) or more of the
- outstanding shares, or (iii) beneficial ownership of such entity.
-
- "You" (or "Your") shall mean an individual or Legal Entity
- exercising permissions granted by this License.
-
- "Source" form shall mean the preferred form for making modifications,
- including but not limited to software source code, documentation
- source, and configuration files.
-
- "Object" form shall mean any form resulting from mechanical
- transformation or translation of a Source form, including but
- not limited to compiled object code, generated documentation,
- and conversions to other media types.
-
- "Work" shall mean the work of authorship, whether in Source or
- Object form, made available under the License, as indicated by a
- copyright notice that is included in or attached to the work
- (an example is provided in the Appendix below).
-
- "Derivative Works" shall mean any work, whether in Source or Object
- form, that is based on (or derived from) the Work and for which the
- editorial revisions, annotations, elaborations, or other modifications
- represent, as a whole, an original work of authorship. For the purposes
- of this License, Derivative Works shall not include works that remain
- separable from, or merely link (or bind by name) to the interfaces of,
- the Work and Derivative Works thereof.
-
- "Contribution" shall mean any work of authorship, including
- the original version of the Work and any modifications or additions
- to that Work or Derivative Works thereof, that is intentionally
- submitted to Licensor for inclusion in the Work by the copyright owner
- or by an individual or Legal Entity authorized to submit on behalf of
- the copyright owner. For the purposes of this definition, "submitted"
- means any form of electronic, verbal, or written communication sent
- to the Licensor or its representatives, including but not limited to
- communication on electronic mailing lists, source code control systems,
- and issue tracking systems that are managed by, or on behalf of, the
- Licensor for the purpose of discussing and improving the Work, but
- excluding communication that is conspicuously marked or otherwise
- designated in writing by the copyright owner as "Not a Contribution."
-
- "Contributor" shall mean Licensor and any individual or Legal Entity
- on behalf of whom a Contribution has been received by Licensor and
- subsequently incorporated within the Work.
-
- 2. Grant of Copyright License. Subject to the terms and conditions of
- this License, each Contributor hereby grants to You a perpetual,
- worldwide, non-exclusive, no-charge, royalty-free, irrevocable
- copyright license to reproduce, prepare Derivative Works of,
- publicly display, publicly perform, sublicense, and distribute the
- Work and such Derivative Works in Source or Object form.
-
- 3. Grant of Patent License. Subject to the terms and conditions of
- this License, each Contributor hereby grants to You a perpetual,
- worldwide, non-exclusive, no-charge, royalty-free, irrevocable
- (except as stated in this section) patent license to make, have made,
- use, offer to sell, sell, import, and otherwise transfer the Work,
- where such license applies only to those patent claims licensable
- by such Contributor that are necessarily infringed by their
- Contribution(s) alone or by combination of their Contribution(s)
- with the Work to which such Contribution(s) was submitted. If You
- institute patent litigation against any entity (including a
- cross-claim or counterclaim in a lawsuit) alleging that the Work
- or a Contribution incorporated within the Work constitutes direct
- or contributory patent infringement, then any patent licenses
- granted to You under this License for that Work shall terminate
- as of the date such litigation is filed.
-
- 4. Redistribution. You may reproduce and distribute copies of the
- Work or Derivative Works thereof in any medium, with or without
- modifications, and in Source or Object form, provided that You
- meet the following conditions:
-
- (a) You must give any other recipients of the Work or
- Derivative Works a copy of this License; and
-
- (b) You must cause any modified files to carry prominent notices
- stating that You changed the files; and
-
- (c) You must retain, in the Source form of any Derivative Works
- that You distribute, all copyright, patent, trademark, and
- attribution notices from the Source form of the Work,
- excluding those notices that do not pertain to any part of
- the Derivative Works; and
-
- (d) If the Work includes a "NOTICE" text file as part of its
- distribution, then any Derivative Works that You distribute must
- include a readable copy of the attribution notices contained
- within such NOTICE file, excluding those notices that do not
- pertain to any part of the Derivative Works, in at least one
- of the following places: within a NOTICE text file distributed
- as part of the Derivative Works; within the Source form or
- documentation, if provided along with the Derivative Works; or,
- within a display generated by the Derivative Works, if and
- wherever such third-party notices normally appear. The contents
- of the NOTICE file are for informational purposes only and
- do not modify the License. You may add Your own attribution
- notices within Derivative Works that You distribute, alongside
- or as an addendum to the NOTICE text from the Work, provided
- that such additional attribution notices cannot be construed
- as modifying the License.
-
- You may add Your own copyright statement to Your modifications and
- may provide additional or different license terms and conditions
- for use, reproduction, or distribution of Your modifications, or
- for any such Derivative Works as a whole, provided Your use,
- reproduction, and distribution of the Work otherwise complies with
- the conditions stated in this License.
-
- 5. Submission of Contributions. Unless You explicitly state otherwise,
- any Contribution intentionally submitted for inclusion in the Work
- by You to the Licensor shall be under the terms and conditions of
- this License, without any additional terms or conditions.
- Notwithstanding the above, nothing herein shall supersede or modify
- the terms of any separate license agreement you may have executed
- with Licensor regarding such Contributions.
-
- 6. Trademarks. This License does not grant permission to use the trade
- names, trademarks, service marks, or product names of the Licensor,
- except as required for reasonable and customary use in describing the
- origin of the Work and reproducing the content of the NOTICE file.
-
- 7. Disclaimer of Warranty. Unless required by applicable law or
- agreed to in writing, Licensor provides the Work (and each
- Contributor provides its Contributions) on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
- implied, including, without limitation, any warranties or conditions
- of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
- PARTICULAR PURPOSE. You are solely responsible for determining the
- appropriateness of using or redistributing the Work and assume any
- risks associated with Your exercise of permissions under this License.
-
- 8. Limitation of Liability. In no event and under no legal theory,
- whether in tort (including negligence), contract, or otherwise,
- unless required by applicable law (such as deliberate and grossly
- negligent acts) or agreed to in writing, shall any Contributor be
- liable to You for damages, including any direct, indirect, special,
- incidental, or consequential damages of any character arising as a
- result of this License or out of the use or inability to use the
- Work (including but not limited to damages for loss of goodwill,
- work stoppage, computer failure or malfunction, or any and all
- other commercial damages or losses), even if such Contributor
- has been advised of the possibility of such damages.
-
- 9. Accepting Warranty or Additional Liability. While redistributing
- the Work or Derivative Works thereof, You may choose to offer,
- and charge a fee for, acceptance of support, warranty, indemnity,
- or other liability obligations and/or rights consistent with this
- License. However, in accepting such obligations, You may act only
- on Your own behalf and on Your sole responsibility, not on behalf
- of any other Contributor, and only if You agree to indemnify,
- defend, and hold each Contributor harmless for any liability
- incurred by, or claims asserted against, such Contributor by reason
- of your accepting any such warranty or additional liability.
-
- END OF TERMS AND CONDITIONS
-
- APPENDIX: How to apply the Apache License to your work.
-
- To apply the Apache License to your work, attach the following
- boilerplate notice, with the fields enclosed by brackets "[]"
- replaced with your own identifying information. (Don't include
- the brackets!) The text should be enclosed in the appropriate
- comment syntax for the file format. We also recommend that a
- file or class name and description of purpose be included on the
- same "printed page" as the copyright notice for easier
- identification within third-party archives.
-
- Copyright [yyyy] [name of copyright owner]
-
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
-
-
----- LLVM Exceptions to the Apache 2.0 License ----
-
-As an exception, if, as a result of your compiling your source code, portions
-of this Software are embedded into an Object form of such source code, you
-may redistribute such embedded portions in such Object form without complying
-with the conditions of Sections 4(a), 4(b) and 4(d) of the License.
-
-In addition, if you combine or link compiled forms of this Software with
-software that is licensed under the GPLv2 ("Combined Software") and if a
-court of competent jurisdiction determines that the patent provision (Section
-3), the indemnity provision (Section 9) or other Section of the License
-conflicts with the conditions of the GPLv2, you may retroactively and
-prospectively choose to deem waived or otherwise exclude such Section(s) of
-the License, but only in their entirety and only with respect to the Combined
-Software.
-
-==============================================================================
-Software from third parties included in the LLVM Project:
-==============================================================================
-The LLVM Project contains third party software which is under different license
-terms. All such code will be identified clearly using at least one of two
-mechanisms:
-1) It will be in a separate directory tree with its own `LICENSE.txt` or
- `LICENSE` file at the top containing the specific license and restrictions
- which apply to that software, or
-2) It will contain specific license and restriction terms at the top of every
- file.
-
---------------------------------------------------------------------------------
-
-3rdparty dependency gRPC is statically linked in certain binary
-distributions, like the python wheels. gRPC has the following license:
-
-Copyright 2014 gRPC authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
---------------------------------------------------------------------------------
-
-3rdparty dependency Apache Thrift is statically linked in certain binary
-distributions, like the python wheels. Apache Thrift has the following license:
-
-Apache Thrift
-Copyright (C) 2006 - 2019, The Apache Software Foundation
-
-This product includes software developed at
-The Apache Software Foundation (http://www.apache.org/).
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
---------------------------------------------------------------------------------
-
-3rdparty dependency Apache ORC is statically linked in certain binary
-distributions, like the python wheels. Apache ORC has the following license:
-
-Apache ORC
-Copyright 2013-2019 The Apache Software Foundation
-
-This product includes software developed by The Apache Software
-Foundation (http://www.apache.org/).
-
-This product includes software developed by Hewlett-Packard:
-(c) Copyright [2014-2015] Hewlett-Packard Development Company, L.P
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
---------------------------------------------------------------------------------
-
-3rdparty dependency zstd is statically linked in certain binary
-distributions, like the python wheels. ZSTD has the following license:
-
-BSD License
-
-For Zstandard software
-
-Copyright (c) 2016-present, Facebook, Inc. All rights reserved.
-
-Redistribution and use in source and binary forms, with or without modification,
-are permitted provided that the following conditions are met:
-
- * Redistributions of source code must retain the above copyright notice, this
- list of conditions and the following disclaimer.
-
- * Redistributions in binary form must reproduce the above copyright notice,
- this list of conditions and the following disclaimer in the documentation
- and/or other materials provided with the distribution.
-
- * Neither the name Facebook nor the names of its contributors may be used to
- endorse or promote products derived from this software without specific
- prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
-ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
-ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
-ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
---------------------------------------------------------------------------------
-
-3rdparty dependency lz4 is statically linked in certain binary
-distributions, like the python wheels. lz4 has the following license:
-
-LZ4 Library
-Copyright (c) 2011-2016, Yann Collet
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without modification,
-are permitted provided that the following conditions are met:
-
-* Redistributions of source code must retain the above copyright notice, this
- list of conditions and the following disclaimer.
-
-* Redistributions in binary form must reproduce the above copyright notice, this
- list of conditions and the following disclaimer in the documentation and/or
- other materials provided with the distribution.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
-ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
-ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
-ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
---------------------------------------------------------------------------------
-
-3rdparty dependency Brotli is statically linked in certain binary
-distributions, like the python wheels. Brotli has the following license:
-
-Copyright (c) 2009, 2010, 2013-2016 by the Brotli Authors.
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-THE SOFTWARE.
-
---------------------------------------------------------------------------------
-
-3rdparty dependency rapidjson is statically linked in certain binary
-distributions, like the python wheels. rapidjson and its dependencies have the
-following licenses:
-
-Tencent is pleased to support the open source community by making RapidJSON
-available.
-
-Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip.
-All rights reserved.
-
-If you have downloaded a copy of the RapidJSON binary from Tencent, please note
-that the RapidJSON binary is licensed under the MIT License.
-If you have downloaded a copy of the RapidJSON source code from Tencent, please
-note that RapidJSON source code is licensed under the MIT License, except for
-the third-party components listed below which are subject to different license
-terms. Your integration of RapidJSON into your own projects may require
-compliance with the MIT License, as well as the other licenses applicable to
-the third-party components included within RapidJSON. To avoid the problematic
-JSON license in your own projects, it's sufficient to exclude the
-bin/jsonchecker/ directory, as it's the only code under the JSON license.
-A copy of the MIT License is included in this file.
-
-Other dependencies and licenses:
-
- Open Source Software Licensed Under the BSD License:
- --------------------------------------------------------------------
-
- The msinttypes r29
- Copyright (c) 2006-2013 Alexander Chemeris
- All rights reserved.
-
- Redistribution and use in source and binary forms, with or without
- modification, are permitted provided that the following conditions are met:
-
- * Redistributions of source code must retain the above copyright notice,
- this list of conditions and the following disclaimer.
- * Redistributions in binary form must reproduce the above copyright notice,
- this list of conditions and the following disclaimer in the documentation
- and/or other materials provided with the distribution.
- * Neither the name of copyright holder nor the names of its contributors
- may be used to endorse or promote products derived from this software
- without specific prior written permission.
-
- THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND ANY
- EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- DISCLAIMED. IN NO EVENT SHALL THE REGENTS AND CONTRIBUTORS BE LIABLE FOR
- ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
- SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
- CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
- DAMAGE.
-
- Open Source Software Licensed Under the JSON License:
- --------------------------------------------------------------------
-
- json.org
- Copyright (c) 2002 JSON.org
- All Rights Reserved.
-
- JSON_checker
- Copyright (c) 2002 JSON.org
- All Rights Reserved.
-
-
- Terms of the JSON License:
- ---------------------------------------------------
-
- Permission is hereby granted, free of charge, to any person obtaining a
- copy of this software and associated documentation files (the "Software"),
- to deal in the Software without restriction, including without limitation
- the rights to use, copy, modify, merge, publish, distribute, sublicense,
- and/or sell copies of the Software, and to permit persons to whom the
- Software is furnished to do so, subject to the following conditions:
-
- The above copyright notice and this permission notice shall be included in
- all copies or substantial portions of the Software.
-
- The Software shall be used for Good, not Evil.
-
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
- DEALINGS IN THE SOFTWARE.
-
-
- Terms of the MIT License:
- --------------------------------------------------------------------
-
- Permission is hereby granted, free of charge, to any person obtaining a
- copy of this software and associated documentation files (the "Software"),
- to deal in the Software without restriction, including without limitation
- the rights to use, copy, modify, merge, publish, distribute, sublicense,
- and/or sell copies of the Software, and to permit persons to whom the
- Software is furnished to do so, subject to the following conditions:
-
- The above copyright notice and this permission notice shall be included
- in all copies or substantial portions of the Software.
-
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
- DEALINGS IN THE SOFTWARE.
-
---------------------------------------------------------------------------------
-
-3rdparty dependency snappy is statically linked in certain binary
-distributions, like the python wheels. snappy has the following license:
-
-Copyright 2011, Google Inc.
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are
-met:
-
- * Redistributions of source code must retain the above copyright notice,
- this list of conditions and the following disclaimer.
- * Redistributions in binary form must reproduce the above copyright notice,
- this list of conditions and the following disclaimer in the documentation
- and/or other materials provided with the distribution.
- * Neither the name of Google Inc. nor the names of its contributors may be
- used to endorse or promote products derived from this software without
- specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-===
-
-Some of the benchmark data in testdata/ is licensed differently:
-
- - fireworks.jpeg is Copyright 2013 Steinar H. Gunderson, and
- is licensed under the Creative Commons Attribution 3.0 license
- (CC-BY-3.0). See https://creativecommons.org/licenses/by/3.0/
- for more information.
-
- - kppkn.gtb is taken from the Gaviota chess tablebase set, and
- is licensed under the MIT License. See
- https://sites.google.com/site/gaviotachessengine/Home/endgame-tablebases-1
- for more information.
-
- - paper-100k.pdf is an excerpt (bytes 92160 to 194560) from the paper
- “Combinatorial Modeling of Chromatin Features Quantitatively Predicts DNA
- Replication Timing in _Drosophila_” by Federico Comoglio and Renato Paro,
- which is licensed under the CC-BY license. See
- http://www.ploscompbiol.org/static/license for more ifnormation.
-
- - alice29.txt, asyoulik.txt, plrabn12.txt and lcet10.txt are from Project
- Gutenberg. The first three have expired copyrights and are in the public
- domain; the latter does not have expired copyright, but is still in the
- public domain according to the license information
- (http://www.gutenberg.org/ebooks/53).
-
---------------------------------------------------------------------------------
-
-3rdparty dependency gflags is statically linked in certain binary
-distributions, like the python wheels. gflags has the following license:
-
-Copyright (c) 2006, Google Inc.
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are
-met:
-
- * Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
- * Redistributions in binary form must reproduce the above
-copyright notice, this list of conditions and the following disclaimer
-in the documentation and/or other materials provided with the
-distribution.
- * Neither the name of Google Inc. nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
---------------------------------------------------------------------------------
-
-3rdparty dependency glog is statically linked in certain binary
-distributions, like the python wheels. glog has the following license:
-
-Copyright (c) 2008, Google Inc.
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are
-met:
-
- * Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
- * Redistributions in binary form must reproduce the above
-copyright notice, this list of conditions and the following disclaimer
-in the documentation and/or other materials provided with the
-distribution.
- * Neither the name of Google Inc. nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-
-A function gettimeofday in utilities.cc is based on
-
-http://www.google.com/codesearch/p?hl=en#dR3YEbitojA/COPYING&q=GetSystemTimeAsFileTime%20license:bsd
-
-The license of this code is:
-
-Copyright (c) 2003-2008, Jouni Malinen <j...@w1.fi> and contributors
-All Rights Reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are
-met:
-
-1. Redistributions of source code must retain the above copyright
- notice, this list of conditions and the following disclaimer.
-
-2. Redistributions in binary form must reproduce the above copyright
- notice, this list of conditions and the following disclaimer in the
- documentation and/or other materials provided with the distribution.
-
-3. Neither the name(s) of the above-listed copyright holder(s) nor the
- names of its contributors may be used to endorse or promote products
- derived from this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
---------------------------------------------------------------------------------
-
-3rdparty dependency re2 is statically linked in certain binary
-distributions, like the python wheels. re2 has the following license:
-
-Copyright (c) 2009 The RE2 Authors. All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are
-met:
-
- * Redistributions of source code must retain the above copyright
- notice, this list of conditions and the following disclaimer.
- * Redistributions in binary form must reproduce the above
- copyright notice, this list of conditions and the following
- disclaimer in the documentation and/or other materials provided
- with the distribution.
- * Neither the name of Google Inc. nor the names of its contributors
- may be used to endorse or promote products derived from this
- software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
---------------------------------------------------------------------------------
-
-3rdparty dependency c-ares is statically linked in certain binary
-distributions, like the python wheels. c-ares has the following license:
-
-# c-ares license
-
-Copyright (c) 2007 - 2018, Daniel Stenberg with many contributors, see AUTHORS
-file.
-
-Copyright 1998 by the Massachusetts Institute of Technology.
-
-Permission to use, copy, modify, and distribute this software and its
-documentation for any purpose and without fee is hereby granted, provided that
-the above copyright notice appear in all copies and that both that copyright
-notice and this permission notice appear in supporting documentation, and that
-the name of M.I.T. not be used in advertising or publicity pertaining to
-distribution of the software without specific, written prior permission.
-M.I.T. makes no representations about the suitability of this software for any
-purpose. It is provided "as is" without express or implied warranty.
-
---------------------------------------------------------------------------------
-
-3rdparty dependency zlib is redistributed as a dynamically linked shared
-library in certain binary distributions, like the python wheels. In the future
-this will likely change to static linkage. zlib has the following license:
-
-zlib.h -- interface of the 'zlib' general purpose compression library
- version 1.2.11, January 15th, 2017
-
- Copyright (C) 1995-2017 Jean-loup Gailly and Mark Adler
-
- This software is provided 'as-is', without any express or implied
- warranty. In no event will the authors be held liable for any damages
- arising from the use of this software.
-
- Permission is granted to anyone to use this software for any purpose,
- including commercial applications, and to alter it and redistribute it
- freely, subject to the following restrictions:
-
- 1. The origin of this software must not be misrepresented; you must not
- claim that you wrote the original software. If you use this software
- in a product, an acknowledgment in the product documentation would be
- appreciated but is not required.
- 2. Altered source versions must be plainly marked as such, and must not be
- misrepresented as being the original software.
- 3. This notice may not be removed or altered from any source distribution.
-
- Jean-loup Gailly Mark Adler
- jloup@gzip.org madler@alumni.caltech.edu
-
---------------------------------------------------------------------------------
-
-3rdparty dependency openssl is redistributed as a dynamically linked shared
-library in certain binary distributions, like the python wheels. openssl
-preceding version 3 has the following license:
-
- LICENSE ISSUES
- ==============
-
- The OpenSSL toolkit stays under a double license, i.e. both the conditions of
- the OpenSSL License and the original SSLeay license apply to the toolkit.
- See below for the actual license texts.
-
- OpenSSL License
- ---------------
-
-/* ====================================================================
- * Copyright (c) 1998-2019 The OpenSSL Project. All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- *
- * 1. Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- *
- * 2. Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in
- * the documentation and/or other materials provided with the
- * distribution.
- *
- * 3. All advertising materials mentioning features or use of this
- * software must display the following acknowledgment:
- * "This product includes software developed by the OpenSSL Project
- * for use in the OpenSSL Toolkit. (http://www.openssl.org/)"
- *
- * 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to
- * endorse or promote products derived from this software without
- * prior written permission. For written permission, please contact
- * openssl-core@openssl.org.
- *
- * 5. Products derived from this software may not be called "OpenSSL"
- * nor may "OpenSSL" appear in their names without prior written
- * permission of the OpenSSL Project.
- *
- * 6. Redistributions of any form whatsoever must retain the following
- * acknowledgment:
- * "This product includes software developed by the OpenSSL Project
- * for use in the OpenSSL Toolkit (http://www.openssl.org/)"
- *
- * THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY
- * EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
- * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE OpenSSL PROJECT OR
- * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
- * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
- * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
- * OF THE POSSIBILITY OF SUCH DAMAGE.
- * ====================================================================
- *
- * This product includes cryptographic software written by Eric Young
- * (eay@cryptsoft.com). This product includes software written by Tim
- * Hudson (tjh@cryptsoft.com).
- *
- */
-
- Original SSLeay License
- -----------------------
-
-/* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
- * All rights reserved.
- *
- * This package is an SSL implementation written
- * by Eric Young (eay@cryptsoft.com).
- * The implementation was written so as to conform with Netscapes SSL.
- *
- * This library is free for commercial and non-commercial use as long as
- * the following conditions are aheared to. The following conditions
- * apply to all code found in this distribution, be it the RC4, RSA,
- * lhash, DES, etc., code; not just the SSL code. The SSL documentation
- * included with this distribution is covered by the same copyright terms
- * except that the holder is Tim Hudson (tjh@cryptsoft.com).
- *
- * Copyright remains Eric Young's, and as such any Copyright notices in
- * the code are not to be removed.
- * If this package is used in a product, Eric Young should be given attribution
- * as the author of the parts of the library used.
- * This can be in the form of a textual message at program startup or
- * in documentation (online or textual) provided with the package.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the copyright
- * notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in the
- * documentation and/or other materials provided with the distribution.
- * 3. All advertising materials mentioning features or use of this software
- * must display the following acknowledgement:
- * "This product includes cryptographic software written by
- * Eric Young (eay@cryptsoft.com)"
- * The word 'cryptographic' can be left out if the rouines from the library
- * being used are not cryptographic related :-).
- * 4. If you include any Windows specific code (or a derivative thereof) from
- * the apps directory (application code) you must include an acknowledgement:
- * "This product includes software written by Tim Hudson (tjh@cryptsoft.com)"
- *
- * THIS SOFTWARE IS PROVIDED BY ERIC YOUNG ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * The licence and distribution terms for any publically available version or
- * derivative of this code cannot be changed. i.e. this code cannot simply be
- * copied and put under another distribution licence
- * [including the GNU Public Licence.]
- */
-
---------------------------------------------------------------------------------
-
-This project includes code from the rtools-backports project.
-
-* ci/scripts/PKGBUILD and ci/scripts/r_windows_build.sh are based on code
- from the rtools-backports project.
-
-Copyright: Copyright (c) 2013 - 2019, Алексей and Jeroen Ooms.
-All rights reserved.
-Homepage: https://github.com/r-windows/rtools-backports
-License: 3-clause BSD
-
---------------------------------------------------------------------------------
-
-Some code from pandas has been adapted for the pyarrow codebase. pandas is
-available under the 3-clause BSD license, which follows:
-
-pandas license
-==============
-
-Copyright (c) 2011-2012, Lambda Foundry, Inc. and PyData Development Team
-All rights reserved.
-
-Copyright (c) 2008-2011 AQR Capital Management, LLC
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are
-met:
-
- * Redistributions of source code must retain the above copyright
- notice, this list of conditions and the following disclaimer.
-
- * Redistributions in binary form must reproduce the above
- copyright notice, this list of conditions and the following
- disclaimer in the documentation and/or other materials provided
- with the distribution.
-
- * Neither the name of the copyright holder nor the names of any
- contributors may be used to endorse or promote products derived
- from this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER AND CONTRIBUTORS
-"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
---------------------------------------------------------------------------------
-
-Some bits from DyND, in particular aspects of the build system, have been
-adapted from libdynd and dynd-python under the terms of the BSD 2-clause
-license
-
-The BSD 2-Clause License
-
- Copyright (C) 2011-12, Dynamic NDArray Developers
- All rights reserved.
-
- Redistribution and use in source and binary forms, with or without
- modification, are permitted provided that the following conditions are
- met:
-
- * Redistributions of source code must retain the above copyright
- notice, this list of conditions and the following disclaimer.
-
- * Redistributions in binary form must reproduce the above
- copyright notice, this list of conditions and the following
- disclaimer in the documentation and/or other materials provided
- with the distribution.
-
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-Dynamic NDArray Developers list:
-
- * Mark Wiebe
- * Continuum Analytics
-
---------------------------------------------------------------------------------
-
-Some source code from Ibis (https://github.com/cloudera/ibis) has been adapted
-for PyArrow. Ibis is released under the Apache License, Version 2.0.
-
---------------------------------------------------------------------------------
-
-This project includes code from the autobrew project.
-
-* r/tools/autobrew and dev/tasks/homebrew-formulae/autobrew/apache-arrow.rb
- are based on code from the autobrew project.
-
-Copyright (c) 2019, Jeroen Ooms
-License: MIT
-Homepage: https://github.com/jeroen/autobrew
-
---------------------------------------------------------------------------------
-
-dev/tasks/homebrew-formulae/apache-arrow.rb has the following license:
-
-BSD 2-Clause License
-
-Copyright (c) 2009-present, Homebrew contributors
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-
-* Redistributions of source code must retain the above copyright notice, this
- list of conditions and the following disclaimer.
-
-* Redistributions in binary form must reproduce the above copyright notice,
- this list of conditions and the following disclaimer in the documentation
- and/or other materials provided with the distribution.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
-FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
-SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
-CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
-OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-----------------------------------------------------------------------
-
-cpp/src/arrow/vendored/base64.cpp has the following license
-
-ZLIB License
-
-Copyright (C) 2004-2017 René Nyffenegger
-
-This source code is provided 'as-is', without any express or implied
-warranty. In no event will the author be held liable for any damages arising
-from the use of this software.
-
-Permission is granted to anyone to use this software for any purpose, including
-commercial applications, and to alter it and redistribute it freely, subject to
-the following restrictions:
-
-1. The origin of this source code must not be misrepresented; you must not
- claim that you wrote the original source code. If you use this source code
- in a product, an acknowledgment in the product documentation would be
- appreciated but is not required.
-
-2. Altered source versions must be plainly marked as such, and must not be
- misrepresented as being the original source code.
-
-3. This notice may not be removed or altered from any source distribution.
-
-René Nyffenegger rene.nyffenegger@adp-gmbh.ch
-
---------------------------------------------------------------------------------
-
-The file cpp/src/arrow/vendored/optional.hpp has the following license
-
-Boost Software License - Version 1.0 - August 17th, 2003
-
-Permission is hereby granted, free of charge, to any person or organization
-obtaining a copy of the software and accompanying documentation covered by
-this license (the "Software") to use, reproduce, display, distribute,
-execute, and transmit the Software, and to prepare derivative works of the
-Software, and to permit third-parties to whom the Software is furnished to
-do so, all subject to the following:
-
-The copyright notices in the Software and this entire statement, including
-the above license grant, this restriction and the following disclaimer,
-must be included in all copies of the Software, in whole or in part, and
-all derivative works of the Software, unless such copies or derivative
-works are solely in the form of machine-executable object code generated by
-a source language processor.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
-SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
-FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
-ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
-DEALINGS IN THE SOFTWARE.
-
---------------------------------------------------------------------------------
-
-This project includes code from Folly.
-
- * cpp/src/arrow/vendored/ProducerConsumerQueue.h
-
-is based on Folly's
-
- * folly/Portability.h
- * folly/lang/Align.h
- * folly/ProducerConsumerQueue.h
-
-Copyright: Copyright (c) Facebook, Inc. and its affiliates.
-Home page: https://github.com/facebook/folly
-License: http://www.apache.org/licenses/LICENSE-2.0
-
---------------------------------------------------------------------------------
-
-The file cpp/src/arrow/vendored/musl/strptime.c has the following license
-
-Copyright © 2005-2020 Rich Felker, et al.
-
-Permission is hereby granted, free of charge, to any person obtaining
-a copy of this software and associated documentation files (the
-"Software"), to deal in the Software without restriction, including
-without limitation the rights to use, copy, modify, merge, publish,
-distribute, sublicense, and/or sell copies of the Software, and to
-permit persons to whom the Software is furnished to do so, subject to
-the following conditions:
-
-The above copyright notice and this permission notice shall be
-included in all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
-IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
-CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
-TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
-SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-
---------------------------------------------------------------------------------
-
-The file cpp/cmake_modules/BuildUtils.cmake contains code from
-
-https://gist.github.com/cristianadam/ef920342939a89fae3e8a85ca9459b49
-
-which is made available under the MIT license
-
-Copyright (c) 2019 Cristian Adam
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.
-
---------------------------------------------------------------------------------
-
-The files in cpp/src/arrow/vendored/portable-snippets/ contain code from
-
-https://github.com/nemequ/portable-snippets
-
-and have the following copyright notice:
-
-Each source file contains a preamble explaining the license situation
-for that file, which takes priority over this file. With the
-exception of some code pulled in from other repositories (such as
-µnit, an MIT-licensed project which is used for testing), the code is
-public domain, released using the CC0 1.0 Universal dedication (*).
-
-(*) https://creativecommons.org/publicdomain/zero/1.0/legalcode
-
---------------------------------------------------------------------------------
-
-The files in cpp/src/arrow/vendored/fast_float/ contain code from
-
-https://github.com/lemire/fast_float
-
-which is made available under the Apache License 2.0.
-
---------------------------------------------------------------------------------
-
-The file python/pyarrow/vendored/version.py contains code from
-
-https://github.com/pypa/packaging/
-
-which is made available under both the Apache license v2.0 and the
-BSD 2-clause license.
diff --git a/README.md b/README.md
deleted file mode 100644
index 5d6a58a..0000000
--- a/README.md
+++ /dev/null
@@ -1,24 +0,0 @@
-<!---
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
--->
-
-# Experimental safe parquet
-
-This repository contains experimental code to read from parquet safely.
-
-This is _not_ officially supported code from Apache Arrow.
diff --git a/dev/.gitignore b/dev/.gitignore
deleted file mode 100644
index c03a7c7..0000000
--- a/dev/.gitignore
+++ /dev/null
@@ -1,22 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# Python virtual environments for dev tools
-.venv*/
-
-__pycache__
-*.egg-info
diff --git a/dev/README.md b/dev/README.md
deleted file mode 100644
index b4ea02b..0000000
--- a/dev/README.md
+++ /dev/null
@@ -1,57 +0,0 @@
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~ with the License. You may obtain a copy of the License at
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing,
- ~ software distributed under the License is distributed on an
- ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ~ KIND, either express or implied. See the License for the
- ~ specific language governing permissions and limitations
- ~ under the License.
- -->
-
-# Arrow Developer Scripts
-
-This directory contains scripts useful to developers when packaging,
-testing, or committing to Arrow.
-
-## Verifying Release Candidates
-
-We have provided a script to assist with verifying release candidates:
-
-```shell
-bash dev/release/verify-release-candidate.sh 0.7.0 0
-```
-
-Currently this only works on Linux (patches to expand to macOS welcome!). Read
-the script for information about system dependencies.
-
-On Windows, we have a script that verifies C++ and Python (requires Visual
-Studio 2015):
-
-```
-dev/release/verify-release-candidate.bat apache-arrow-0.7.0.tar.gz
-```
-
-### Verifying the JavaScript release
-
-For JavaScript-specific releases, use a different verification script:
-
-```shell
-bash dev/release/js-verify-release-candidate.sh 0.7.0 0
-```
-
-# Integration testing
-
-Build the following base image used by multiple tests:
-
-```shell
-docker build -t arrow_integration_xenial_base -f docker_common/Dockerfile.xenial.base .
-```
diff --git a/dev/release/README.md b/dev/release/README.md
deleted file mode 100644
index 5b31083..0000000
--- a/dev/release/README.md
+++ /dev/null
@@ -1,266 +0,0 @@
-<!---
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
--->
-
-# Release Process
-
-## Branching
-
-We would maintain two branches: `active_release` and `master`.
-
-- All new PRs are created and merged against `master`
-- All versions are created from the `active_release` branch
-- Once merged to master, changes are "cherry-picked" (via a hopefully soon to be automated process), to the `active_release` branch based on the judgement of the original PR author and maintainers.
-
-- We do not merge breaking api changes, as defined in [Rust RFC 1105](https://github.com/rust-lang/rfcs/blob/master/text/1105-api-evolution.md) to the `active_release`
-
-Please see the [original proposal](https://docs.google.com/document/d/1tMQ67iu8XyGGZuj--h9WQYB9inCk6c2sL_4xMTwENGc/edit?ts=60961758) document the rational of this change.
-
-## Release Branching
-
-We aim to release every other week from the `active_release` branch.
-
-Every other Monday, a maintainer proposes a minor (e.g. `4.1.0` to `4.2.0`) or patch (e.g `4.1.0` to `4.1.1`) release, depending on changes to the `active_release` in the previous 2 weeks, following the process beloe.
-
-If this release is approved by at least three PMC members, a new version from that tarball is released to crates.io later in the week.
-
-Apache Arrow in general does synchronized major releases every three months. The Rust implementation aims to do its major releases in the same time frame.
-
-# Release Mechanics
-
-This directory contains the scripts used to manage an Apache Arrow Release.
-
-# Process Overview
-
-As part of the Apache governance model, official releases consist of
-signed source tarballs approved by the PMC.
-
-We then use the code in the approved source tarball to release to
-crates.io, the Rust ecosystem's package manager.
-
-## Branching
-
-# Release Preparation
-
-# Change Log
-
-We create a `CHANGELOG.md` so our users know what has been changed between releases.
-
-The CHANGELOG is created automatically using
-[change_log.sh](https://github.com/apache/arrow-rs/blob/master/change_log.sh)
-
-This script creates a changelog using github issues and the
-labels associated with them.
-
-# Mechanics of creating a release
-
-## Prepare the release branch and tags
-
-First, ensure that `active_release` contains the content of the desired release. For minor and patch releases, no additional steps are needed.
-
-To prepare for _a major release_, change `active release` to point at the latest `master` with commands such as:
-
-```
-git checkout active_release
-git fetch apache
-git reset --hard apache/master
-git push -f
-```
-
-### Update CHANGELOG.md + Version
-
-Now prepare a PR to update `CHANGELOG.md` and versions on `active_release` branch to reflect the planned release.
-
-See [#298](https://github.com/apache/arrow-rs/pull/298) for an example.
-
-Here are the commands used to prepare the 4.1.0 release:
-
-```bash
-git checkout active_release
-git pull
-git checkout -b make-release
-
-# manully edit ./dev/release/update_change_log.sh to reflect the release version
-# create the changelog
-CHANGELOG_GITHUB_TOKEN=<TOKEN> ./dev/release/update_change_log.sh
-# review change log / edit issues and labels if needed, rerun
-git commit -a -m 'Create changelog'
-
-# update versions
-sed -i '' -e 's/5.0.0-SNAPSHOT/4.1.0/g' `find . -name 'Cargo.toml'`
-git commit -a -m 'Update version'
-```
-
-Note that when reviewing the change log, rather than editing the
-`CHANGELOG.md`, it is preferred to update the issues and their labels
-(e.g. add `invalid` label to exclude them from release notes)
-
-## Prepare release candidate tarball
-
-(Note you need to be a committer to run these scripts as they upload to the apache svn distribution servers)
-
-### Create git tag for the release:
-
-While the official release artifact is a signed tarball, we also tag the commit it was created for convenience and code archaeology.
-
-Using a string such as `4.0.1` as the `<version>`, create and push the tag thusly:
-
-```shell
-git fetch apache
-git tag <version> apache/active_release
-# push tag to apache
-git push apache <version>
-```
-
-### Pick an Release Candidate (RC) number
-
-Pick numbers in sequential order, with `1` for `rc1`, `2` for `rc2`, etc.
-
-### Create, sign, and upload tarball
-
-Run the `create-tarball.sh` with the `<version>` tag and `<rc>` and you found in previous steps:
-
-```shell
-./dev/release/create-tarball.sh 4.1.0 2
-```
-
-This script
-
-1. creates and uploads a release candidate tarball to the [arrow
- dev](https://dist.apache.org/repos/dist/dev/arrow) location on the
- apache distribution svn server
-
-2. provide you an email template to
- send to dev@arrow.apache.org for release voting.
-
-### Vote on Release Candidate tarball
-
-Send the email output from the script to dev@arrow.apache.org. The email should look like
-
-```
-To: dev@arrow.apache.org
-Subject: [VOTE][RUST] Release Apache Arrow
-
-Hi,
-
-I would like to propose a release of Apache Arrow Rust
-Implementation, version 4.1.0.
-
-This release candidate is based on commit: a5dd428f57e62db20a945e8b1895de91405958c4 [1]
-
-The proposed release tarball and signatures are hosted at [2].
-The changelog is located at [3].
-
-Please download, verify checksums and signatures, run the unit tests,
-and vote on the release.
-
-The vote will be open for at least 72 hours.
-
-[ ] +1 Release this as Apache Arrow Rust
-[ ] +0
-[ ] -1 Do not release this as Apache Arrow Rust because...
-
-[1]: https://github.com/apache/arrow-rs/tree/a5dd428f57e62db20a945e8b1895de91405958c4
-[2]: https://dist.apache.org/repos/dist/dev/arrow/apache-arrow-rs-4.1.0
-[3]: https://github.com/apache/arrow-rs/blob/a5dd428f57e62db20a945e8b1895de91405958c4/CHANGELOG.md
-```
-
-For the release to become "official" it needs at least three PMC members to vote +1 on it.
-
-#### Verifying Release Candidates
-
-There is a script in this repository which can be used to help `dev/release/verify-release-candidate.sh` assist the verification process. Run it like:
-
-```
-./dev/release/verify-release-candidate.sh 4.1.0 2
-```
-
-#### If the release is not approved
-
-If the release is not approved, fix whatever the problem is and try again with the next RC number
-
-### If the release is approved,
-
-Move tarball to the release location in SVN, e.g. https://dist.apache.org/repos/dist/release/arrow/arrow-4.1.0/, using the `release-tarball.sh` script:
-
-```shell
-./dev/release/release-tarball.sh 4.1.0 2
-```
-
-### Publish on Crates.io
-
-Only approved releases of the tarball should be published to
-crates.io, in order to conform to Apache Software Foundation
-governance standards.
-
-An Arrow committer can publish this crate after an official project release has
-been made to crates.io using the following instructions.
-
-Follow [these
-instructions](https://doc.rust-lang.org/cargo/reference/publishing.html) to
-create an account and login to crates.io before asking to be added as an owner
-of the [arrow crate](https://crates.io/crates/arrow).
-
-Download and unpack the official release tarball
-
-Verify that the Cargo.toml in the tarball contains the correct version
-(e.g. `version = "0.11.0"`) and then publish the crate with the
-following commands
-
-```shell
-(cd arrow && cargo publish)
-(cd arrow_flight && cargo publish)
-(cd parquet && cargo publish)
-(cd parquet_derive && cargo publish)
-```
-
-# Backporting
-
-As of now, the plan for backporting to `active_release` is to do so semi-manually.
-
-Step 1: Pick the commit to cherry-pick
-
-Step 2: Create cherry-pick PR to active_release
-
-Step 3a: If CI passes, merge cherry-pick PR
-
-Step 3b: If CI doesn't pass or some other changes are needed, the PR should be reviewed / approved as normal prior to merge
-
-For example, to backport `b2de5446cc1e45a0559fb39039d0545df1ac0d26` to active_release use the folliwing
-
-```shell
-git clone git@github.com:apache/arrow-rs.git /tmp/arrow-rs
-
-ARROW_GITHUB_API_TOKEN=$ARROW_GITHUB_API_TOKEN CHECKOUT_ROOT=/tmp/arrow-rs CHERRY_PICK_SHA=b2de5446cc1e45a0559fb39039d0545df1ac0d26 python3 dev/release/cherry-pick-pr.py
-```
-
-## Tags
-
-There are two tags that help keep track of backporting:
-
-1. [`cherry-picked`](https://github.com/apache/arrow-rs/labels/cherry-picked) for PRs that have been cherry-picked/backported to `active_release`
-2. [`release-cherry-pick`](https://github.com/apache/arrow-rs/labels/release-cherry-pick) for the PRs that are the cherry pick
-
-You can find candidates to cherry pick using [this filter](https://github.com/apache/arrow-rs/pulls?q=is%3Apr+is%3Aclosed+-label%3Arelease-cherry-pick+-label%3Acherry-picked)
-
-## Rationale for creating PRs:
-
-1. PRs are a natural place to run the CI tests to make sure there are no logical conflicts
-2. PRs offer a place for the original author / committers to comment and say it should/should not be backported.
-3. PRs offer a way to make cleanups / fixups and approve (if needed) for non cherry pick PRs
-4. There is an additional control / review when the candidate release is created
diff --git a/dev/release/check-rat-report.py b/dev/release/check-rat-report.py
deleted file mode 100644
index e30d72b..0000000
--- a/dev/release/check-rat-report.py
+++ /dev/null
@@ -1,59 +0,0 @@
-#!/usr/bin/python
-##############################################################################
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-##############################################################################
-import fnmatch
-import re
-import sys
-import xml.etree.ElementTree as ET
-
-if len(sys.argv) != 3:
- sys.stderr.write("Usage: %s exclude_globs.lst rat_report.xml\n" %
- sys.argv[0])
- sys.exit(1)
-
-exclude_globs_filename = sys.argv[1]
-xml_filename = sys.argv[2]
-
-globs = [line.strip() for line in open(exclude_globs_filename, "r")]
-
-tree = ET.parse(xml_filename)
-root = tree.getroot()
-resources = root.findall('resource')
-
-all_ok = True
-for r in resources:
- approvals = r.findall('license-approval')
- if not approvals or approvals[0].attrib['name'] == 'true':
- continue
- clean_name = re.sub('^[^/]+/', '', r.attrib['name'])
- excluded = False
- for g in globs:
- if fnmatch.fnmatch(clean_name, g):
- excluded = True
- break
- if not excluded:
- sys.stdout.write("NOT APPROVED: %s (%s): %s\n" % (
- clean_name, r.attrib['name'], approvals[0].attrib['name']))
- all_ok = False
-
-if not all_ok:
- sys.exit(1)
-
-print('OK')
-sys.exit(0)
diff --git a/dev/release/cherry-pick-pr.py b/dev/release/cherry-pick-pr.py
deleted file mode 100755
index 2886a0d..0000000
--- a/dev/release/cherry-pick-pr.py
+++ /dev/null
@@ -1,154 +0,0 @@
-#!/usr/bin/python3
-##############################################################################
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-##############################################################################
-
-# This script is designed to create a cherry pick PR to a target branch
-#
-# Usage: python3 cherry_pick_pr.py
-#
-# To test locally:
-#
-# git clone git@github.com:apache/arrow-rs.git /tmp/arrow-rs
-#
-# pip3 install PyGithub
-# ARROW_GITHUB_API_TOKEN=<..>
-# CHECKOUT_ROOT=<path>
-# CHERRY_PICK_SHA=<sha> python3 cherry-pick-pr.py
-#
-import os
-import sys
-import six
-import subprocess
-
-from pathlib import Path
-
-TARGET_BRANCH = 'active_release'
-TARGET_REPO = 'apache/arrow-rs'
-
-p = Path(__file__)
-
-# Use github workspace if specified
-repo_root = os.environ.get("CHECKOUT_ROOT")
-if repo_root is None:
- print("arrow-rs checkout must be supplied in CHECKOUT_ROOT environment")
- sys.exit(1)
-
-print("Using checkout in {}".format(repo_root))
-
-token = os.environ.get('ARROW_GITHUB_API_TOKEN', None)
-if token is None:
- print("GITHUB token must be supplied in ARROW_GITHUB_API_TOKEN environmet")
- sys.exit(1)
-
-new_sha = os.environ.get('CHERRY_PICK_SHA', None)
-if new_sha is None:
- print("SHA to cherry pick must be supplied in CHERRY_PICK_SHA environment")
- sys.exit(1)
-
-
-# from merge_pr.py from arrow repo
-def run_cmd(cmd):
- if isinstance(cmd, six.string_types):
- cmd = cmd.split(' ')
- try:
- output = subprocess.check_output(cmd)
- except subprocess.CalledProcessError as e:
- # this avoids hiding the stdout / stderr of failed processes
- print('Command failed: %s' % cmd)
- print('With output:')
- print('--------------')
- print(e.output)
- print('--------------')
- raise e
-
- if isinstance(output, six.binary_type):
- output = output.decode('utf-8')
-
- return output
-
-
-os.chdir(repo_root)
-new_sha_short = run_cmd("git rev-parse --short {}".format(new_sha)).strip()
-new_branch = 'cherry_pick_{}'.format(new_sha_short)
-
-
-def make_cherry_pick():
- if os.environ.get('GITHUB_SHA', None) is not None:
- print("Running on github runner, setting email/username")
- run_cmd(['git', 'config', 'user.email', 'dev@arrow.apache.com'])
- run_cmd(['git', 'config', 'user.name', 'Arrow-RS Automation'])
-
- #
- # Create a new branch from active_release
- # and cherry pick to there.
- #
-
- print("Creating cherry pick from {} to {}".format(
- new_sha_short, new_branch
- ))
-
- # The following tortured dance is required due to how the github
- # actions/checkout works (it doesn't pull other branches and pulls
- # only one commit back)
-
- # pull 10 commits back so we can get the proper cherry pick
- # (probably only need 2 but 10 must be better, right?)
- run_cmd(['git', 'fetch', '--depth', '10', 'origin', 'master'])
- run_cmd(['git', 'fetch', 'origin', 'active_release'])
- run_cmd(['git', 'checkout', '-b', new_branch])
- run_cmd(['git', 'reset', '--hard', 'origin/active_release'])
- run_cmd(['git', 'cherry-pick', new_sha])
- run_cmd(['git', 'push', '-u', 'origin', new_branch])
-
-
-
-def make_cherry_pick_pr():
- from github import Github
- g = Github(token)
- repo = g.get_repo(TARGET_REPO)
-
- release_cherry_pick_label = repo.get_label('release-cherry-pick')
- cherry_picked_label = repo.get_label('cherry-picked')
-
- # Default titles
- new_title = 'Cherry pick {} to active_release'.format(new_sha)
- new_commit_message = 'Automatic cherry-pick of {}\n'.format(new_sha)
-
- # try and get info from github api
- commit = repo.get_commit(new_sha)
- for orig_pull in commit.get_pulls():
- new_commit_message += '* Originally appeared in {}: {}\n'.format(
- orig_pull.html_url, orig_pull.title)
- new_title = 'Cherry pick {} to active_release'.format(orig_pull.title)
- orig_pull.add_to_labels(cherry_picked_label)
-
- pr = repo.create_pull(title=new_title,
- body=new_commit_message,
- base='refs/heads/active_release',
- head='refs/heads/{}'.format(new_branch),
- maintainer_can_modify=True,
- )
-
- pr.add_to_labels(release_cherry_pick_label)
-
- print('Created PR {}'.format(pr.html_url))
-
-
-make_cherry_pick()
-make_cherry_pick_pr()
diff --git a/dev/release/create-tarball.sh b/dev/release/create-tarball.sh
deleted file mode 100755
index 2311519..0000000
--- a/dev/release/create-tarball.sh
+++ /dev/null
@@ -1,123 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-
-# This script creates a signed tarball in
-# dev/dist/apache-arrow-rs-<version>-<sha>.tar.gz and uploads it to
-# the "dev" area of the dist.apache.arrow repository and prepares an
-# email for sending to the dev@arrow.apache.org list for a formal
-# vote.
-#
-# See release/README.md for full release instructions
-#
-# Requirements:
-#
-# 1. gpg setup for signing and have uploaded your public
-# signature to https://pgp.mit.edu/
-#
-# 2. Logged into the apache svn server with the appropriate
-# credentials
-#
-#
-# Based in part on 02-source.sh from apache/arrow
-#
-
-set -e
-
-SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-SOURCE_TOP_DIR="$(cd "${SOURCE_DIR}/../../" && pwd)"
-
-if [ "$#" -ne 2 ]; then
- echo "Usage: $0 <tag> <rc>"
- echo "ex. $0 4.1.0 2"
- exit
-fi
-
-tag=$1
-rc=$2
-
-release_hash=$(cd "${SOURCE_TOP_DIR}" && git rev-list --max-count=1 ${tag})
-
-release=apache-arrow-rs-${tag}
-distdir=${SOURCE_TOP_DIR}/dev/dist/${release}-rc${rc}
-tarname=${release}.tar.gz
-tarball=${distdir}/${tarname}
-url="https://dist.apache.org/repos/dist/dev/arrow/${release}-rc${rc}"
-
-echo "Attempting to create ${tarball} from tag ${tag}"
-
-
-if [ -z "$release_hash" ]; then
- echo "Cannot continue: unknown git tag: $tag"
-fi
-
-echo "Draft email for dev@arrow.apache.org mailing list"
-echo ""
-echo "---------------------------------------------------------"
-cat <<MAIL
-To: dev@arrow.apache.org
-Subject: [VOTE][RUST] Release Apache Arrow Rust ${tag} RC${rc}
-
-Hi,
-
-I would like to propose a release of Apache Arrow Rust Implementation, version ${tag}.
-
-This release candidate is based on commit: ${release_hash} [1]
-
-The proposed release tarball and signatures are hosted at [2].
-
-The changelog is located at [3].
-
-Please download, verify checksums and signatures, run the unit tests,
-and vote on the release.
-
-The vote will be open for at least 72 hours.
-
-[ ] +1 Release this as Apache Arrow Rust ${version}
-[ ] +0
-[ ] -1 Do not release this as Apache Arrow Rust ${version} because...
-
-[1]: https://github.com/apache/arrow-rs/tree/${release_hash}
-[2]: ${url}
-[3]: https://github.com/apache/arrow-rs/blob/${release_hash}/CHANGELOG.md
-MAIL
-echo "---------------------------------------------------------"
-
-
-# create <tarball> containing the files in git at $release_hash
-# the files in the tarball are prefixed with {tag} (e.g. 4.0.1)
-mkdir -p ${distdir}
-(cd "${SOURCE_TOP_DIR}" && git archive ${release_hash} --prefix ${release}/ | gzip > ${tarball})
-
-echo "Running rat license checker on ${tarball}"
-${SOURCE_DIR}/run-rat.sh ${tarball}
-
-echo "Signing tarball and creating checksums"
-gpg --armor --output ${tarball}.asc --detach-sig ${tarball}
-# create signing with relative path of tarball
-# so that they can be verified with a command such as
-# shasum --check apache-arrow-rs-4.1.0-rc2.tar.gz.sha512
-(cd ${distdir} && shasum -a 256 ${tarname}) > ${tarball}.sha256
-(cd ${distdir} && shasum -a 512 ${tarname}) > ${tarball}.sha512
-
-echo "Uploading to apache dist/dev to ${url}"
-svn co --depth=empty https://dist.apache.org/repos/dist/dev/arrow ${SOURCE_TOP_DIR}/dev/dist
-svn add ${distdir}
-svn ci -m "Apache Arrow Rust ${tag} ${rc}" ${distdir}
diff --git a/dev/release/rat_exclude_files.txt b/dev/release/rat_exclude_files.txt
deleted file mode 100644
index d2b5aad..0000000
--- a/dev/release/rat_exclude_files.txt
+++ /dev/null
@@ -1,27 +0,0 @@
-*.npmrc
-*.gitignore
-.gitmodules
-*.csv
-*.json
-*.snap
-.github/ISSUE_TEMPLATE/*.md
-.github/pull_request_template.md
-CHANGELOG.md
-pax_global_header
-MANIFEST.in
-requirements.txt
-*.html
-*.sgml
-*.css
-*.png
-*.ico
-*.svg
-*.devhelp2
-*.scss
-.gitattributes
-arrow/test/data/*.csv
-rust-toolchain
-arrow-flight/src/arrow.flight.protocol.rs
-ballista/rust/benchmarks/tpch/queries/q*.sql
-ballista/rust/scheduler/testdata/*
-ballista/ui/scheduler/yarn.lock
diff --git a/dev/release/release-tarball.sh b/dev/release/release-tarball.sh
deleted file mode 100755
index 9612921..0000000
--- a/dev/release/release-tarball.sh
+++ /dev/null
@@ -1,72 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-# This script copies a tarball from the "dev" area of the
-# dist.apache.arrow repository to the "release" area
-#
-# This script should only be run after the release has been approved
-# by the arrow PMC committee.
-#
-# See release/README.md for full release instructions
-#
-# Based in part on post-01-upload.sh from apache/arrow
-
-
-set -e
-set -u
-
-if [ "$#" -ne 2 ]; then
- echo "Usage: $0 <version> <rc-num>"
- echo "ex. $0 4.1.0 2"
- exit
-fi
-
-version=$1
-rc=$2
-
-tmp_dir=tmp-apache-arrow-dist
-
-echo "Recreate temporary directory: ${tmp_dir}"
-rm -rf ${tmp_dir}
-mkdir -p ${tmp_dir}
-
-echo "Clone dev dist repository"
-svn \
- co \
- https://dist.apache.org/repos/dist/dev/arrow/apache-arrow-rs-${version}-rc${rc} \
- ${tmp_dir}/dev
-
-echo "Clone release dist repository"
-svn co https://dist.apache.org/repos/dist/release/arrow ${tmp_dir}/release
-
-echo "Copy ${version}-rc${rc} to release working copy"
-release_version=arrow-rs-${version}
-mkdir -p ${tmp_dir}/release/${release_version}
-cp -r ${tmp_dir}/dev/* ${tmp_dir}/release/${release_version}/
-svn add ${tmp_dir}/release/${release_version}
-
-echo "Commit release"
-svn ci -m "Apache Arrow Rust ${version}" ${tmp_dir}/release
-
-echo "Clean up"
-rm -rf ${tmp_dir}
-
-echo "Success! The release is available here:"
-echo " https://dist.apache.org/repos/dist/release/arrow/${release_version}"
diff --git a/dev/release/run-rat.sh b/dev/release/run-rat.sh
deleted file mode 100755
index 94fa55f..0000000
--- a/dev/release/run-rat.sh
+++ /dev/null
@@ -1,43 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-RAT_VERSION=0.13
-
-# download apache rat
-if [ ! -f apache-rat-${RAT_VERSION}.jar ]; then
- curl -s https://repo1.maven.org/maven2/org/apache/rat/apache-rat/${RAT_VERSION}/apache-rat-${RAT_VERSION}.jar > apache-rat-${RAT_VERSION}.jar
-fi
-
-RAT="java -jar apache-rat-${RAT_VERSION}.jar -x "
-
-RELEASE_DIR=$(cd "$(dirname "$BASH_SOURCE")"; pwd)
-
-# generate the rat report
-$RAT $1 > rat.txt
-python $RELEASE_DIR/check-rat-report.py $RELEASE_DIR/rat_exclude_files.txt rat.txt > filtered_rat.txt
-cat filtered_rat.txt
-UNAPPROVED=`cat filtered_rat.txt | grep "NOT APPROVED" | wc -l`
-
-if [ "0" -eq "${UNAPPROVED}" ]; then
- echo "No unapproved licenses"
-else
- echo "${UNAPPROVED} unapproved licences. Check rat report: rat.txt"
- exit 1
-fi
diff --git a/dev/release/update_change_log.sh b/dev/release/update_change_log.sh
deleted file mode 100755
index a11f966..0000000
--- a/dev/release/update_change_log.sh
+++ /dev/null
@@ -1,39 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-# invokes the changelog generator with the config located in
-# arrow-rs/.github_changelog_generator
-#
-# Usage:
-# CHANGELOG_GITHUB_TOKEN=<TOKEN> ./update_change_log.sh
-
-set -e
-
-SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-SOURCE_TOP_DIR="$(cd "${SOURCE_DIR}/../../" && pwd)"
-
-pushd ${SOURCE_TOP_DIR}
-docker run -it --rm -e CHANGELOG_GITHUB_TOKEN=$CHANGELOG_GITHUB_TOKEN -v "$(pwd)":/usr/local/src/your-app githubchangeloggenerator/github-changelog-generator \
- --user apache \
- --project arrow-rs \
- --since-commit 2021-04-20 \
- --future-release 4.1.0
-
-sed -i "s/\\\n/\n\n/" CHANGELOG.md
diff --git a/dev/release/verify-release-candidate.sh b/dev/release/verify-release-candidate.sh
deleted file mode 100755
index 6c265fb..0000000
--- a/dev/release/verify-release-candidate.sh
+++ /dev/null
@@ -1,153 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-case $# in
- 2) VERSION="$1"
- RC_NUMBER="$2"
- ;;
- *) echo "Usage: $0 X.Y.Z RC_NUMBER"
- exit 1
- ;;
-esac
-
-set -e
-set -x
-set -o pipefail
-
-SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)"
-ARROW_DIR="$(dirname $(dirname ${SOURCE_DIR}))"
-ARROW_DIST_URL='https://dist.apache.org/repos/dist/dev/arrow'
-
-download_dist_file() {
- curl \
- --silent \
- --show-error \
- --fail \
- --location \
- --remote-name $ARROW_DIST_URL/$1
-}
-
-download_rc_file() {
- download_dist_file apache-arrow-rs-${VERSION}-rc${RC_NUMBER}/$1
-}
-
-import_gpg_keys() {
- download_dist_file KEYS
- gpg --import KEYS
-}
-
-fetch_archive() {
- local dist_name=$1
- download_rc_file ${dist_name}.tar.gz
- download_rc_file ${dist_name}.tar.gz.asc
- download_rc_file ${dist_name}.tar.gz.sha256
- download_rc_file ${dist_name}.tar.gz.sha512
- gpg --verify ${dist_name}.tar.gz.asc ${dist_name}.tar.gz
- shasum -a 256 -c ${dist_name}.tar.gz.sha256
- shasum -a 512 -c ${dist_name}.tar.gz.sha512
-}
-
-verify_dir_artifact_signatures() {
- # verify the signature and the checksums of each artifact
- find $1 -name '*.asc' | while read sigfile; do
- artifact=${sigfile/.asc/}
- gpg --verify $sigfile $artifact || exit 1
-
- # go into the directory because the checksum files contain only the
- # basename of the artifact
- pushd $(dirname $artifact)
- base_artifact=$(basename $artifact)
- if [ -f $base_artifact.sha256 ]; then
- shasum -a 256 -c $base_artifact.sha256 || exit 1
- fi
- shasum -a 512 -c $base_artifact.sha512 || exit 1
- popd
- done
-}
-
-setup_tempdir() {
- cleanup() {
- if [ "${TEST_SUCCESS}" = "yes" ]; then
- rm -fr "${ARROW_TMPDIR}"
- else
- echo "Failed to verify release candidate. See ${ARROW_TMPDIR} for details."
- fi
- }
-
- if [ -z "${ARROW_TMPDIR}" ]; then
- # clean up automatically if ARROW_TMPDIR is not defined
- ARROW_TMPDIR=$(mktemp -d -t "$1.XXXXX")
- trap cleanup EXIT
- else
- # don't clean up automatically
- mkdir -p "${ARROW_TMPDIR}"
- fi
-}
-
-test_source_distribution() {
- # install rust toolchain in a similar fashion like test-miniconda
- export RUSTUP_HOME=$PWD/test-rustup
- export CARGO_HOME=$PWD/test-rustup
-
- curl https://sh.rustup.rs -sSf | sh -s -- -y --no-modify-path
-
- export PATH=$RUSTUP_HOME/bin:$PATH
- source $RUSTUP_HOME/env
-
- # build and test rust
-
- # raises on any formatting errors
- rustup component add rustfmt --toolchain stable
- cargo fmt --all -- --check
-
- # Clone testing repositories if not cloned already
- git clone https://github.com/apache/arrow-testing.git arrow-testing-data
- git clone https://github.com/apache/parquet-testing.git parquet-testing-data
- export ARROW_TEST_DATA=$PWD/arrow-testing-data/data
- export PARQUET_TEST_DATA=$PWD/parquet-testing-data/data
-
- # use local modules because we don't publish modules to crates.io yet
- sed \
- -i.bak \
- -E \
- -e 's/^parquet = "([^"]*)"/parquet = { version = "\1", path = "..\/parquet" }/g' \
- */Cargo.toml
-
- cargo build
- cargo test
-}
-
-TEST_SUCCESS=no
-
-setup_tempdir "arrow-${VERSION}"
-echo "Working in sandbox ${ARROW_TMPDIR}"
-cd ${ARROW_TMPDIR}
-
-dist_name="apache-arrow-rs-${VERSION}"
-import_gpg_keys
-fetch_archive ${dist_name}
-tar xf ${dist_name}.tar.gz
-pushd ${dist_name}
-test_source_distribution
-popd
-
-TEST_SUCCESS=yes
-echo 'Release candidate looks good!'
-exit 0
diff --git a/header b/header
deleted file mode 100644
index 70665d1..0000000
--- a/header
+++ /dev/null
@@ -1,16 +0,0 @@
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
-http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
diff --git a/parquet-testing b/parquet-testing
deleted file mode 160000
index 8e7badc..0000000
--- a/parquet-testing
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit 8e7badc6a3817a02e06d17b5d8ab6b6dc356e890
diff --git a/parquet/Cargo.toml b/parquet/Cargo.toml
deleted file mode 100644
index cfe4e41..0000000
--- a/parquet/Cargo.toml
+++ /dev/null
@@ -1,82 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-[package]
-name = "parquet"
-version = "5.0.0-SNAPSHOT"
-license = "Apache-2.0"
-description = "Apache Parquet implementation in Rust"
-homepage = "https://github.com/apache/arrow-rs"
-repository = "https://github.com/apache/arrow-rs"
-authors = ["Apache Arrow <de...@arrow.apache.org>"]
-keywords = [ "arrow", "parquet", "hadoop" ]
-readme = "README.md"
-build = "build.rs"
-edition = "2018"
-
-[dependencies]
-# update note: pin `parquet-format` to specific version until it does not break at minor
-# version, see ARROW-11187.
-parquet-format = "~2.6.1"
-byteorder = "1"
-thrift = "0.13"
-snap = { version = "1.0", optional = true }
-brotli = { version = "3.3", optional = true }
-flate2 = { version = "1.0", optional = true }
-lz4 = { version = "1.23", optional = true }
-zstd = { version = "0.8", optional = true }
-chrono = "0.4"
-num-bigint = "0.4"
-arrow = { version = "4.3.0", optional = true }
-base64 = { version = "0.13", optional = true }
-clap = { version = "2.33.3", optional = true }
-serde_json = { version = "1.0", features = ["preserve_order"], optional = true }
-rand = "0.8"
-
-[dev-dependencies]
-criterion = "0.3"
-rand = "0.8"
-snap = "1.0"
-brotli = "3.3"
-flate2 = "1.0"
-lz4 = "1.23"
-zstd = "0.8"
-serde_json = { version = "1.0", features = ["preserve_order"] }
-
-[features]
-default = ["arrow", "snap", "brotli", "flate2", "lz4", "zstd", "base64"]
-cli = ["serde_json", "base64", "clap"]
-
-[[ bin ]]
-name = "parquet-read"
-required-features = ["cli"]
-
-[[ bin ]]
-name = "parquet-schema"
-required-features = ["cli"]
-
-[[ bin ]]
-name = "parquet-rowcount"
-required-features = ["cli"]
-
-[[bench]]
-name = "arrow_writer"
-harness = false
-
-[[bench]]
-name = "arrow_array_reader"
-harness = false
\ No newline at end of file
diff --git a/parquet/README.md b/parquet/README.md
deleted file mode 100644
index 338dba7..0000000
--- a/parquet/README.md
+++ /dev/null
@@ -1,143 +0,0 @@
-<!---
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
--->
-
-# An Apache Parquet implementation in Rust
-
-[](https://crates.io/crates/parquet)
-
-## Usage
-
-Add this to your Cargo.toml:
-
-```toml
-[dependencies]
-parquet = "^4"
-```
-
-and this to your crate root:
-
-```rust
-extern crate parquet;
-```
-
-Example usage of reading data:
-
-```rust
-use std::fs::File;
-use std::path::Path;
-use parquet::file::reader::{FileReader, SerializedFileReader};
-
-let file = File::open(&Path::new("/path/to/file")).unwrap();
-let reader = SerializedFileReader::new(file).unwrap();
-let mut iter = reader.get_row_iter(None).unwrap();
-while let Some(record) = iter.next() {
- println!("{}", record);
-}
-```
-
-See [crate documentation](https://docs.rs/crate/parquet/5.0.0-SNAPSHOT) on available API.
-
-## Upgrading from versions prior to 4.0
-
-If you are upgrading from version 3.0 or previous of this crate, you
-likely need to change your code to use [`ConvertedType`] rather than
-[`LogicalType`] to preserve existing behaviour in your code.
-
-Version 2.4.0 of the Parquet format introduced a `LogicalType` to replace the existing `ConvertedType`.
-This crate used `parquet::basic::LogicalType` to map to the `ConvertedType`, but this has been renamed to `parquet::basic::ConvertedType` from version 4.0 of this crate.
-
-The `ConvertedType` is deprecated in the format, but is still written
-to preserve backward compatibility.
-It is preferred that `LogicalType` is used, as it supports nanosecond
-precision timestamps without using the deprecated `Int96` Parquet type.
-
-## Supported Parquet Version
-
-- Parquet-format 2.6.0
-
-To update Parquet format to a newer version, check if [parquet-format](https://github.com/sunchao/parquet-format-rs)
-version is available. Then simply update version of `parquet-format` crate in Cargo.toml.
-
-## Features
-
-- [x] All encodings supported
-- [x] All compression codecs supported
-- [x] Read support
- - [x] Primitive column value readers
- - [x] Row record reader
- - [x] Arrow record reader
-- [ ] Statistics support
-- [x] Write support
- - [x] Primitive column value writers
- - [ ] Row record writer
- - [x] Arrow record writer
-- [ ] Predicate pushdown
-- [x] Parquet format 2.6.0 support
-
-## Requirements
-
-Parquet requires LLVM. Our windows CI image includes LLVM but to build the libraries locally windows
-users will have to install LLVM. Follow [this](https://github.com/appveyor/ci/issues/2651) link for info.
-
-## Build
-
-Run `cargo build` or `cargo build --release` to build in release mode.
-Some features take advantage of SSE4.2 instructions, which can be
-enabled by adding `RUSTFLAGS="-C target-feature=+sse4.2"` before the
-`cargo build` command.
-
-## Test
-
-Run `cargo test` for unit tests. To also run tests related to the binaries, use `cargo test --features cli`.
-
-## Binaries
-
-The following binaries are provided (use `cargo install --features cli` to install them):
-
-- **parquet-schema** for printing Parquet file schema and metadata.
- `Usage: parquet-schema <file-path>`, where `file-path` is the path to a Parquet file. Use `-v/--verbose` flag
- to print full metadata or schema only (when not specified only schema will be printed).
-
-- **parquet-read** for reading records from a Parquet file.
- `Usage: parquet-read <file-path> [num-records]`, where `file-path` is the path to a Parquet file,
- and `num-records` is the number of records to read from a file (when not specified all records will
- be printed). Use `-j/--json` to print records in JSON lines format.
-
-- **parquet-rowcount** for reporting the number of records in one or more Parquet files.
- `Usage: parquet-rowcount <file-paths>...`, where `<file-paths>...` is a space separated list of one or more
- files to read.
-
-If you see `Library not loaded` error, please make sure `LD_LIBRARY_PATH` is set properly:
-
-```
-export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$(rustc --print sysroot)/lib
-```
-
-## Benchmarks
-
-Run `cargo bench` for benchmarks.
-
-## Docs
-
-To build documentation, run `cargo doc --no-deps`.
-To compile and view in the browser, run `cargo doc --no-deps --open`.
-
-## License
-
-Licensed under the Apache License, Version 2.0: http://www.apache.org/licenses/LICENSE-2.0.
diff --git a/parquet/benches/arrow_array_reader.rs b/parquet/benches/arrow_array_reader.rs
deleted file mode 100644
index 6e87512..0000000
--- a/parquet/benches/arrow_array_reader.rs
+++ /dev/null
@@ -1,766 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-use criterion::{criterion_group, criterion_main, Criterion};
-use parquet::util::{DataPageBuilder, DataPageBuilderImpl, InMemoryPageIterator};
-use parquet::{
- arrow::array_reader::ArrayReader,
- basic::Encoding,
- column::page::PageIterator,
- data_type::{ByteArrayType, Int32Type},
- schema::types::{ColumnDescPtr, SchemaDescPtr},
-};
-use std::{collections::VecDeque, sync::Arc};
-
-fn build_test_schema() -> SchemaDescPtr {
- use parquet::schema::{parser::parse_message_type, types::SchemaDescriptor};
- let message_type = "
- message test_schema {
- REQUIRED INT32 mandatory_int32_leaf;
- REPEATED Group test_mid_int32 {
- OPTIONAL INT32 optional_int32_leaf;
- }
- REQUIRED BYTE_ARRAY mandatory_string_leaf (UTF8);
- REPEATED Group test_mid_string {
- OPTIONAL BYTE_ARRAY optional_string_leaf (UTF8);
- }
- }
- ";
- parse_message_type(message_type)
- .map(|t| Arc::new(SchemaDescriptor::new(Arc::new(t))))
- .unwrap()
-}
-
-// test data params
-const NUM_ROW_GROUPS: usize = 1;
-const PAGES_PER_GROUP: usize = 2;
-const VALUES_PER_PAGE: usize = 10_000;
-const BATCH_SIZE: usize = 8192;
-
-use rand::{rngs::StdRng, Rng, SeedableRng};
-
-pub fn seedable_rng() -> StdRng {
- StdRng::seed_from_u64(42)
-}
-
-fn build_plain_encoded_int32_page_iterator(
- schema: SchemaDescPtr,
- column_desc: ColumnDescPtr,
- null_density: f32,
-) -> impl PageIterator + Clone {
- let max_def_level = column_desc.max_def_level();
- let max_rep_level = column_desc.max_rep_level();
- let rep_levels = vec![0; VALUES_PER_PAGE];
- let mut rng = seedable_rng();
- let mut pages: Vec<Vec<parquet::column::page::Page>> = Vec::new();
- let mut int32_value = 0;
- for _i in 0..NUM_ROW_GROUPS {
- let mut column_chunk_pages = Vec::new();
- for _j in 0..PAGES_PER_GROUP {
- // generate page
- let mut values = Vec::with_capacity(VALUES_PER_PAGE);
- let mut def_levels = Vec::with_capacity(VALUES_PER_PAGE);
- for _k in 0..VALUES_PER_PAGE {
- let def_level = if rng.gen::<f32>() < null_density {
- max_def_level - 1
- } else {
- max_def_level
- };
- if def_level == max_def_level {
- int32_value += 1;
- values.push(int32_value);
- }
- def_levels.push(def_level);
- }
- let mut page_builder =
- DataPageBuilderImpl::new(column_desc.clone(), values.len() as u32, true);
- page_builder.add_rep_levels(max_rep_level, &rep_levels);
- page_builder.add_def_levels(max_def_level, &def_levels);
- page_builder.add_values::<Int32Type>(Encoding::PLAIN, &values);
- column_chunk_pages.push(page_builder.consume());
- }
- pages.push(column_chunk_pages);
- }
-
- InMemoryPageIterator::new(schema, column_desc, pages)
-}
-
-fn build_dictionary_encoded_int32_page_iterator(
- schema: SchemaDescPtr,
- column_desc: ColumnDescPtr,
- null_density: f32,
-) -> impl PageIterator + Clone {
- use parquet::encoding::{DictEncoder, Encoder};
- let max_def_level = column_desc.max_def_level();
- let max_rep_level = column_desc.max_rep_level();
- let rep_levels = vec![0; VALUES_PER_PAGE];
- // generate 1% unique values
- const NUM_UNIQUE_VALUES: usize = VALUES_PER_PAGE / 100;
- let unique_values = (0..NUM_UNIQUE_VALUES)
- .map(|x| (x + 1) as i32)
- .collect::<Vec<_>>();
- let mut rng = seedable_rng();
- let mut pages: Vec<Vec<parquet::column::page::Page>> = Vec::new();
- for _i in 0..NUM_ROW_GROUPS {
- let mut column_chunk_pages = VecDeque::new();
- let mem_tracker = Arc::new(parquet::memory::MemTracker::new());
- let mut dict_encoder =
- DictEncoder::<Int32Type>::new(column_desc.clone(), mem_tracker);
- // add data pages
- for _j in 0..PAGES_PER_GROUP {
- // generate page
- let mut values = Vec::with_capacity(VALUES_PER_PAGE);
- let mut def_levels = Vec::with_capacity(VALUES_PER_PAGE);
- for _k in 0..VALUES_PER_PAGE {
- let def_level = if rng.gen::<f32>() < null_density {
- max_def_level - 1
- } else {
- max_def_level
- };
- if def_level == max_def_level {
- // select random value from list of unique values
- let int32_value = unique_values[rng.gen_range(0..NUM_UNIQUE_VALUES)];
- values.push(int32_value);
- }
- def_levels.push(def_level);
- }
- let mut page_builder =
- DataPageBuilderImpl::new(column_desc.clone(), values.len() as u32, true);
- page_builder.add_rep_levels(max_rep_level, &rep_levels);
- page_builder.add_def_levels(max_def_level, &def_levels);
- let _ = dict_encoder.put(&values);
- let indices = dict_encoder
- .write_indices()
- .expect("write_indices() should be OK");
- page_builder.add_indices(indices);
- column_chunk_pages.push_back(page_builder.consume());
- }
- // add dictionary page
- let dict = dict_encoder
- .write_dict()
- .expect("write_dict() should be OK");
- let dict_page = parquet::column::page::Page::DictionaryPage {
- buf: dict,
- num_values: dict_encoder.num_entries() as u32,
- encoding: Encoding::RLE_DICTIONARY,
- is_sorted: false,
- };
- column_chunk_pages.push_front(dict_page);
- pages.push(column_chunk_pages.into());
- }
-
- InMemoryPageIterator::new(schema, column_desc, pages)
-}
-
-fn build_plain_encoded_string_page_iterator(
- schema: SchemaDescPtr,
- column_desc: ColumnDescPtr,
- null_density: f32,
-) -> impl PageIterator + Clone {
- let max_def_level = column_desc.max_def_level();
- let max_rep_level = column_desc.max_rep_level();
- let rep_levels = vec![0; VALUES_PER_PAGE];
- let mut rng = seedable_rng();
- let mut pages: Vec<Vec<parquet::column::page::Page>> = Vec::new();
- for i in 0..NUM_ROW_GROUPS {
- let mut column_chunk_pages = Vec::new();
- for j in 0..PAGES_PER_GROUP {
- // generate page
- let mut values = Vec::with_capacity(VALUES_PER_PAGE);
- let mut def_levels = Vec::with_capacity(VALUES_PER_PAGE);
- for k in 0..VALUES_PER_PAGE {
- let def_level = if rng.gen::<f32>() < null_density {
- max_def_level - 1
- } else {
- max_def_level
- };
- if def_level == max_def_level {
- let string_value =
- format!("Test value {}, row group: {}, page: {}", k, i, j);
- values
- .push(parquet::data_type::ByteArray::from(string_value.as_str()));
- }
- def_levels.push(def_level);
- }
- let mut page_builder =
- DataPageBuilderImpl::new(column_desc.clone(), values.len() as u32, true);
- page_builder.add_rep_levels(max_rep_level, &rep_levels);
- page_builder.add_def_levels(max_def_level, &def_levels);
- page_builder.add_values::<ByteArrayType>(Encoding::PLAIN, &values);
- column_chunk_pages.push(page_builder.consume());
- }
- pages.push(column_chunk_pages);
- }
-
- InMemoryPageIterator::new(schema, column_desc, pages)
-}
-
-fn build_dictionary_encoded_string_page_iterator(
- schema: SchemaDescPtr,
- column_desc: ColumnDescPtr,
- null_density: f32,
-) -> impl PageIterator + Clone {
- use parquet::encoding::{DictEncoder, Encoder};
- let max_def_level = column_desc.max_def_level();
- let max_rep_level = column_desc.max_rep_level();
- let rep_levels = vec![0; VALUES_PER_PAGE];
- // generate 1% unique values
- const NUM_UNIQUE_VALUES: usize = VALUES_PER_PAGE / 100;
- let unique_values = (0..NUM_UNIQUE_VALUES)
- .map(|x| format!("Dictionary value {}", x))
- .collect::<Vec<_>>();
- let mut rng = seedable_rng();
- let mut pages: Vec<Vec<parquet::column::page::Page>> = Vec::new();
- for _i in 0..NUM_ROW_GROUPS {
- let mut column_chunk_pages = VecDeque::new();
- let mem_tracker = Arc::new(parquet::memory::MemTracker::new());
- let mut dict_encoder =
- DictEncoder::<ByteArrayType>::new(column_desc.clone(), mem_tracker);
- // add data pages
- for _j in 0..PAGES_PER_GROUP {
- // generate page
- let mut values = Vec::with_capacity(VALUES_PER_PAGE);
- let mut def_levels = Vec::with_capacity(VALUES_PER_PAGE);
- for _k in 0..VALUES_PER_PAGE {
- let def_level = if rng.gen::<f32>() < null_density {
- max_def_level - 1
- } else {
- max_def_level
- };
- if def_level == max_def_level {
- // select random value from list of unique values
- let string_value =
- unique_values[rng.gen_range(0..NUM_UNIQUE_VALUES)].as_str();
- values.push(parquet::data_type::ByteArray::from(string_value));
- }
- def_levels.push(def_level);
- }
- let mut page_builder =
- DataPageBuilderImpl::new(column_desc.clone(), values.len() as u32, true);
- page_builder.add_rep_levels(max_rep_level, &rep_levels);
- page_builder.add_def_levels(max_def_level, &def_levels);
- let _ = dict_encoder.put(&values);
- let indices = dict_encoder
- .write_indices()
- .expect("write_indices() should be OK");
- page_builder.add_indices(indices);
- column_chunk_pages.push_back(page_builder.consume());
- }
- // add dictionary page
- let dict = dict_encoder
- .write_dict()
- .expect("write_dict() should be OK");
- let dict_page = parquet::column::page::Page::DictionaryPage {
- buf: dict,
- num_values: dict_encoder.num_entries() as u32,
- encoding: Encoding::RLE_DICTIONARY,
- is_sorted: false,
- };
- column_chunk_pages.push_front(dict_page);
- pages.push(column_chunk_pages.into());
- }
-
- InMemoryPageIterator::new(schema, column_desc, pages)
-}
-
-fn bench_array_reader(mut array_reader: impl ArrayReader) -> usize {
- // test procedure: read data in batches of 8192 until no more data
- let mut total_count = 0;
- loop {
- let array = array_reader.next_batch(BATCH_SIZE);
- let array_len = array.unwrap().len();
- total_count += array_len;
- if array_len < BATCH_SIZE {
- break;
- }
- }
- total_count
-}
-
-fn create_int32_arrow_array_reader(
- page_iterator: impl PageIterator + 'static,
- column_desc: ColumnDescPtr,
-) -> impl ArrayReader {
- use parquet::arrow::arrow_array_reader::{ArrowArrayReader, PrimitiveArrayConverter};
- let converter = PrimitiveArrayConverter::<arrow::datatypes::Int32Type>::new();
- ArrowArrayReader::try_new(page_iterator, column_desc, converter, None).unwrap()
-}
-
-fn create_int32_primitive_array_reader(
- page_iterator: impl PageIterator + 'static,
- column_desc: ColumnDescPtr,
-) -> impl ArrayReader {
- use parquet::arrow::array_reader::PrimitiveArrayReader;
- PrimitiveArrayReader::<Int32Type>::new(Box::new(page_iterator), column_desc, None)
- .unwrap()
-}
-
-fn create_string_arrow_array_reader(
- page_iterator: impl PageIterator + 'static,
- column_desc: ColumnDescPtr,
-) -> impl ArrayReader {
- use parquet::arrow::arrow_array_reader::{ArrowArrayReader, StringArrayConverter};
- let converter = StringArrayConverter::new();
- ArrowArrayReader::try_new(page_iterator, column_desc, converter, None).unwrap()
-}
-
-fn create_string_complex_array_reader(
- page_iterator: impl PageIterator + 'static,
- column_desc: ColumnDescPtr,
-) -> impl ArrayReader {
- use parquet::arrow::array_reader::ComplexObjectArrayReader;
- use parquet::arrow::converter::{Utf8ArrayConverter, Utf8Converter};
- let converter = Utf8Converter::new(Utf8ArrayConverter {});
- ComplexObjectArrayReader::<parquet::data_type::ByteArrayType, Utf8Converter>::new(
- Box::new(page_iterator),
- column_desc,
- converter,
- None,
- )
- .unwrap()
-}
-
-fn add_benches(c: &mut Criterion) {
- const EXPECTED_VALUE_COUNT: usize =
- NUM_ROW_GROUPS * PAGES_PER_GROUP * VALUES_PER_PAGE;
- let mut group = c.benchmark_group("arrow_array_reader");
-
- let mut count: usize = 0;
-
- let schema = build_test_schema();
- let mandatory_int32_column_desc = schema.column(0);
- let optional_int32_column_desc = schema.column(1);
- let mandatory_string_column_desc = schema.column(2);
- // println!("mandatory_string_column_desc: {:?}", mandatory_string_column_desc);
- let optional_string_column_desc = schema.column(3);
- // println!("optional_string_column_desc: {:?}", optional_string_column_desc);
-
- // primitive / int32 benchmarks
- // =============================
-
- // int32, plain encoded, no NULLs
- let plain_int32_no_null_data = build_plain_encoded_int32_page_iterator(
- schema.clone(),
- mandatory_int32_column_desc.clone(),
- 0.0,
- );
- group.bench_function(
- "read Int32Array, plain encoded, mandatory, no NULLs - old",
- |b| {
- b.iter(|| {
- let array_reader = create_int32_primitive_array_reader(
- plain_int32_no_null_data.clone(),
- mandatory_int32_column_desc.clone(),
- );
- count = bench_array_reader(array_reader);
- })
- },
- );
- assert_eq!(count, EXPECTED_VALUE_COUNT);
-
- group.bench_function(
- "read Int32Array, plain encoded, mandatory, no NULLs - new",
- |b| {
- b.iter(|| {
- let array_reader = create_int32_arrow_array_reader(
- plain_int32_no_null_data.clone(),
- mandatory_int32_column_desc.clone(),
- );
- count = bench_array_reader(array_reader);
- })
- },
- );
- assert_eq!(count, EXPECTED_VALUE_COUNT);
-
- let plain_int32_no_null_data = build_plain_encoded_int32_page_iterator(
- schema.clone(),
- optional_int32_column_desc.clone(),
- 0.0,
- );
- group.bench_function(
- "read Int32Array, plain encoded, optional, no NULLs - old",
- |b| {
- b.iter(|| {
- let array_reader = create_int32_primitive_array_reader(
- plain_int32_no_null_data.clone(),
- optional_int32_column_desc.clone(),
- );
- count = bench_array_reader(array_reader);
- })
- },
- );
- assert_eq!(count, EXPECTED_VALUE_COUNT);
-
- group.bench_function(
- "read Int32Array, plain encoded, optional, no NULLs - new",
- |b| {
- b.iter(|| {
- let array_reader = create_int32_arrow_array_reader(
- plain_int32_no_null_data.clone(),
- optional_int32_column_desc.clone(),
- );
- count = bench_array_reader(array_reader);
- })
- },
- );
- assert_eq!(count, EXPECTED_VALUE_COUNT);
-
- // int32, plain encoded, half NULLs
- let plain_int32_half_null_data = build_plain_encoded_int32_page_iterator(
- schema.clone(),
- optional_int32_column_desc.clone(),
- 0.5,
- );
- group.bench_function(
- "read Int32Array, plain encoded, optional, half NULLs - old",
- |b| {
- b.iter(|| {
- let array_reader = create_int32_primitive_array_reader(
- plain_int32_half_null_data.clone(),
- optional_int32_column_desc.clone(),
- );
- count = bench_array_reader(array_reader);
- })
- },
- );
- assert_eq!(count, EXPECTED_VALUE_COUNT);
-
- group.bench_function(
- "read Int32Array, plain encoded, optional, half NULLs - new",
- |b| {
- b.iter(|| {
- let array_reader = create_int32_arrow_array_reader(
- plain_int32_half_null_data.clone(),
- optional_int32_column_desc.clone(),
- );
- count = bench_array_reader(array_reader);
- })
- },
- );
- assert_eq!(count, EXPECTED_VALUE_COUNT);
-
- // int32, dictionary encoded, no NULLs
- let dictionary_int32_no_null_data = build_dictionary_encoded_int32_page_iterator(
- schema.clone(),
- mandatory_int32_column_desc.clone(),
- 0.0,
- );
- group.bench_function(
- "read Int32Array, dictionary encoded, mandatory, no NULLs - old",
- |b| {
- b.iter(|| {
- let array_reader = create_int32_primitive_array_reader(
- dictionary_int32_no_null_data.clone(),
- mandatory_int32_column_desc.clone(),
- );
- count = bench_array_reader(array_reader);
- })
- },
- );
- assert_eq!(count, EXPECTED_VALUE_COUNT);
-
- group.bench_function(
- "read Int32Array, dictionary encoded, mandatory, no NULLs - new",
- |b| {
- b.iter(|| {
- let array_reader = create_int32_arrow_array_reader(
- dictionary_int32_no_null_data.clone(),
- mandatory_int32_column_desc.clone(),
- );
- count = bench_array_reader(array_reader);
- })
- },
- );
- assert_eq!(count, EXPECTED_VALUE_COUNT);
-
- let dictionary_int32_no_null_data = build_dictionary_encoded_int32_page_iterator(
- schema.clone(),
- optional_int32_column_desc.clone(),
- 0.0,
- );
- group.bench_function(
- "read Int32Array, dictionary encoded, optional, no NULLs - old",
- |b| {
- b.iter(|| {
- let array_reader = create_int32_primitive_array_reader(
- dictionary_int32_no_null_data.clone(),
- optional_int32_column_desc.clone(),
- );
- count = bench_array_reader(array_reader);
- })
- },
- );
- assert_eq!(count, EXPECTED_VALUE_COUNT);
-
- group.bench_function(
- "read Int32Array, dictionary encoded, optional, no NULLs - new",
- |b| {
- b.iter(|| {
- let array_reader = create_int32_arrow_array_reader(
- dictionary_int32_no_null_data.clone(),
- optional_int32_column_desc.clone(),
- );
- count = bench_array_reader(array_reader);
- })
- },
- );
- assert_eq!(count, EXPECTED_VALUE_COUNT);
-
- // int32, dictionary encoded, half NULLs
- let dictionary_int32_half_null_data = build_dictionary_encoded_int32_page_iterator(
- schema.clone(),
- optional_int32_column_desc.clone(),
- 0.5,
- );
- group.bench_function(
- "read Int32Array, dictionary encoded, optional, half NULLs - old",
- |b| {
- b.iter(|| {
- let array_reader = create_int32_primitive_array_reader(
- dictionary_int32_half_null_data.clone(),
- optional_int32_column_desc.clone(),
- );
- count = bench_array_reader(array_reader);
- })
- },
- );
- assert_eq!(count, EXPECTED_VALUE_COUNT);
-
- group.bench_function(
- "read Int32Array, dictionary encoded, optional, half NULLs - new",
- |b| {
- b.iter(|| {
- let array_reader = create_int32_arrow_array_reader(
- dictionary_int32_half_null_data.clone(),
- optional_int32_column_desc.clone(),
- );
- count = bench_array_reader(array_reader);
- })
- },
- );
- assert_eq!(count, EXPECTED_VALUE_COUNT);
-
- // string benchmarks
- //==============================
-
- // string, plain encoded, no NULLs
- let plain_string_no_null_data = build_plain_encoded_string_page_iterator(
- schema.clone(),
- mandatory_string_column_desc.clone(),
- 0.0,
- );
- group.bench_function(
- "read StringArray, plain encoded, mandatory, no NULLs - old",
- |b| {
- b.iter(|| {
- let array_reader = create_string_complex_array_reader(
- plain_string_no_null_data.clone(),
- mandatory_string_column_desc.clone(),
- );
- count = bench_array_reader(array_reader);
- })
- },
- );
- assert_eq!(count, EXPECTED_VALUE_COUNT);
-
- group.bench_function(
- "read StringArray, plain encoded, mandatory, no NULLs - new",
- |b| {
- b.iter(|| {
- let array_reader = create_string_arrow_array_reader(
- plain_string_no_null_data.clone(),
- mandatory_string_column_desc.clone(),
- );
- count = bench_array_reader(array_reader);
- })
- },
- );
- assert_eq!(count, EXPECTED_VALUE_COUNT);
-
- let plain_string_no_null_data = build_plain_encoded_string_page_iterator(
- schema.clone(),
- optional_string_column_desc.clone(),
- 0.0,
- );
- group.bench_function(
- "read StringArray, plain encoded, optional, no NULLs - old",
- |b| {
- b.iter(|| {
- let array_reader = create_string_complex_array_reader(
- plain_string_no_null_data.clone(),
- optional_string_column_desc.clone(),
- );
- count = bench_array_reader(array_reader);
- })
- },
- );
- assert_eq!(count, EXPECTED_VALUE_COUNT);
-
- group.bench_function(
- "read StringArray, plain encoded, optional, no NULLs - new",
- |b| {
- b.iter(|| {
- let array_reader = create_string_arrow_array_reader(
- plain_string_no_null_data.clone(),
- optional_string_column_desc.clone(),
- );
- count = bench_array_reader(array_reader);
- })
- },
- );
- assert_eq!(count, EXPECTED_VALUE_COUNT);
-
- // string, plain encoded, half NULLs
- let plain_string_half_null_data = build_plain_encoded_string_page_iterator(
- schema.clone(),
- optional_string_column_desc.clone(),
- 0.5,
- );
- group.bench_function(
- "read StringArray, plain encoded, optional, half NULLs - old",
- |b| {
- b.iter(|| {
- let array_reader = create_string_complex_array_reader(
- plain_string_half_null_data.clone(),
- optional_string_column_desc.clone(),
- );
- count = bench_array_reader(array_reader);
- })
- },
- );
- assert_eq!(count, EXPECTED_VALUE_COUNT);
-
- group.bench_function(
- "read StringArray, plain encoded, optional, half NULLs - new",
- |b| {
- b.iter(|| {
- let array_reader = create_string_arrow_array_reader(
- plain_string_half_null_data.clone(),
- optional_string_column_desc.clone(),
- );
- count = bench_array_reader(array_reader);
- })
- },
- );
- assert_eq!(count, EXPECTED_VALUE_COUNT);
-
- // string, dictionary encoded, no NULLs
- let dictionary_string_no_null_data = build_dictionary_encoded_string_page_iterator(
- schema.clone(),
- mandatory_string_column_desc.clone(),
- 0.0,
- );
- group.bench_function(
- "read StringArray, dictionary encoded, mandatory, no NULLs - old",
- |b| {
- b.iter(|| {
- let array_reader = create_string_complex_array_reader(
- dictionary_string_no_null_data.clone(),
- mandatory_string_column_desc.clone(),
- );
- count = bench_array_reader(array_reader);
- })
- },
- );
- assert_eq!(count, EXPECTED_VALUE_COUNT);
-
- group.bench_function(
- "read StringArray, dictionary encoded, mandatory, no NULLs - new",
- |b| {
- b.iter(|| {
- let array_reader = create_string_arrow_array_reader(
- dictionary_string_no_null_data.clone(),
- mandatory_string_column_desc.clone(),
- );
- count = bench_array_reader(array_reader);
- })
- },
- );
- assert_eq!(count, EXPECTED_VALUE_COUNT);
-
- let dictionary_string_no_null_data = build_dictionary_encoded_string_page_iterator(
- schema.clone(),
- optional_string_column_desc.clone(),
- 0.0,
- );
- group.bench_function(
- "read StringArray, dictionary encoded, optional, no NULLs - old",
- |b| {
- b.iter(|| {
- let array_reader = create_string_complex_array_reader(
- dictionary_string_no_null_data.clone(),
- optional_string_column_desc.clone(),
- );
- count = bench_array_reader(array_reader);
- })
- },
- );
- assert_eq!(count, EXPECTED_VALUE_COUNT);
-
- group.bench_function(
- "read StringArray, dictionary encoded, optional, no NULLs - new",
- |b| {
- b.iter(|| {
- let array_reader = create_string_arrow_array_reader(
- dictionary_string_no_null_data.clone(),
- optional_string_column_desc.clone(),
- );
- count = bench_array_reader(array_reader);
- })
- },
- );
- assert_eq!(count, EXPECTED_VALUE_COUNT);
-
- // string, dictionary encoded, half NULLs
- let dictionary_string_half_null_data = build_dictionary_encoded_string_page_iterator(
- schema,
- optional_string_column_desc.clone(),
- 0.5,
- );
- group.bench_function(
- "read StringArray, dictionary encoded, optional, half NULLs - old",
- |b| {
- b.iter(|| {
- let array_reader = create_string_complex_array_reader(
- dictionary_string_half_null_data.clone(),
- optional_string_column_desc.clone(),
- );
- count = bench_array_reader(array_reader);
- })
- },
- );
- assert_eq!(count, EXPECTED_VALUE_COUNT);
-
- group.bench_function(
- "read StringArray, dictionary encoded, optional, half NULLs - new",
- |b| {
- b.iter(|| {
- let array_reader = create_string_arrow_array_reader(
- dictionary_string_half_null_data.clone(),
- optional_string_column_desc.clone(),
- );
- count = bench_array_reader(array_reader);
- })
- },
- );
- assert_eq!(count, EXPECTED_VALUE_COUNT);
-
- group.finish();
-}
-
-criterion_group!(benches, add_benches);
-criterion_main!(benches);
diff --git a/parquet/benches/arrow_writer.rs b/parquet/benches/arrow_writer.rs
deleted file mode 100644
index 069ed39..0000000
--- a/parquet/benches/arrow_writer.rs
+++ /dev/null
@@ -1,202 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#[macro_use]
-extern crate criterion;
-use criterion::{Criterion, Throughput};
-
-extern crate arrow;
-extern crate parquet;
-
-use std::sync::Arc;
-
-use arrow::datatypes::*;
-use arrow::{record_batch::RecordBatch, util::data_gen::*};
-use parquet::{
- arrow::ArrowWriter, errors::Result, file::writer::InMemoryWriteableCursor,
-};
-
-fn create_primitive_bench_batch(
- size: usize,
- null_density: f32,
- true_density: f32,
-) -> Result<RecordBatch> {
- let fields = vec![
- Field::new("_1", DataType::Int8, true),
- Field::new("_2", DataType::Int16, true),
- Field::new("_3", DataType::Int32, true),
- Field::new("_4", DataType::Int64, true),
- Field::new("_5", DataType::UInt8, true),
- Field::new("_6", DataType::UInt16, true),
- Field::new("_7", DataType::UInt32, true),
- Field::new("_8", DataType::UInt64, true),
- Field::new("_9", DataType::Float32, true),
- Field::new("_10", DataType::Float64, true),
- Field::new("_11", DataType::Date32, true),
- Field::new("_12", DataType::Date64, true),
- Field::new("_13", DataType::Time32(TimeUnit::Second), true),
- Field::new("_14", DataType::Time32(TimeUnit::Millisecond), true),
- Field::new("_15", DataType::Time64(TimeUnit::Microsecond), true),
- Field::new("_16", DataType::Time64(TimeUnit::Nanosecond), true),
- Field::new("_17", DataType::Utf8, true),
- Field::new("_18", DataType::LargeUtf8, true),
- Field::new("_19", DataType::Boolean, true),
- ];
- let schema = Schema::new(fields);
- Ok(create_random_batch(
- Arc::new(schema),
- size,
- null_density,
- true_density,
- )?)
-}
-
-fn _create_nested_bench_batch(
- size: usize,
- null_density: f32,
- true_density: f32,
-) -> Result<RecordBatch> {
- let fields = vec![
- Field::new(
- "_1",
- DataType::Struct(vec![
- Field::new("_1", DataType::Int8, true),
- Field::new(
- "_2",
- DataType::Struct(vec![
- Field::new("_1", DataType::Int8, true),
- Field::new(
- "_1",
- DataType::Struct(vec![
- Field::new("_1", DataType::Int8, true),
- Field::new("_2", DataType::Utf8, true),
- ]),
- true,
- ),
- Field::new("_2", DataType::UInt8, true),
- ]),
- true,
- ),
- ]),
- true,
- ),
- Field::new(
- "_2",
- DataType::LargeList(Box::new(Field::new(
- "item",
- DataType::List(Box::new(Field::new(
- "item",
- DataType::Struct(vec![
- Field::new(
- "_1",
- DataType::Struct(vec![
- Field::new("_1", DataType::Int8, true),
- Field::new("_2", DataType::Int16, true),
- Field::new("_3", DataType::Int32, true),
- ]),
- true,
- ),
- Field::new(
- "_2",
- DataType::List(Box::new(Field::new(
- "",
- DataType::FixedSizeBinary(2),
- true,
- ))),
- true,
- ),
- ]),
- true,
- ))),
- true,
- ))),
- true,
- ),
- ];
- let schema = Schema::new(fields);
- Ok(create_random_batch(
- Arc::new(schema),
- size,
- null_density,
- true_density,
- )?)
-}
-
-#[inline]
-fn write_batch(batch: &RecordBatch) -> Result<()> {
- // Write batch to an in-memory writer
- let cursor = InMemoryWriteableCursor::default();
- let mut writer = ArrowWriter::try_new(cursor, batch.schema(), None)?;
-
- writer.write(&batch)?;
- writer.close()?;
- Ok(())
-}
-
-fn bench_primitive_writer(c: &mut Criterion) {
- let batch = create_primitive_bench_batch(1024, 0.25, 0.75).unwrap();
- let mut group = c.benchmark_group("write_batch primitive");
- group.throughput(Throughput::Bytes(
- batch
- .columns()
- .iter()
- .map(|f| f.get_array_memory_size() as u64)
- .sum(),
- ));
- group.bench_function("1024 values", |b| b.iter(|| write_batch(&batch).unwrap()));
-
- let batch = create_primitive_bench_batch(4096, 0.25, 0.75).unwrap();
- group.throughput(Throughput::Bytes(
- batch
- .columns()
- .iter()
- .map(|f| f.get_array_memory_size() as u64)
- .sum(),
- ));
- group.bench_function("4096 values", |b| b.iter(|| write_batch(&batch).unwrap()));
-
- group.finish();
-}
-
-// This bench triggers a write error, it is ignored for now
-fn _bench_nested_writer(c: &mut Criterion) {
- let batch = _create_nested_bench_batch(1024, 0.25, 0.75).unwrap();
- let mut group = c.benchmark_group("write_batch nested");
- group.throughput(Throughput::Bytes(
- batch
- .columns()
- .iter()
- .map(|f| f.get_array_memory_size() as u64)
- .sum(),
- ));
- group.bench_function("1024 values", |b| b.iter(|| write_batch(&batch).unwrap()));
-
- let batch = create_primitive_bench_batch(4096, 0.25, 0.75).unwrap();
- group.throughput(Throughput::Bytes(
- batch
- .columns()
- .iter()
- .map(|f| f.get_array_memory_size() as u64)
- .sum(),
- ));
- group.bench_function("4096 values", |b| b.iter(|| write_batch(&batch).unwrap()));
-
- group.finish();
-}
-
-criterion_group!(benches, bench_primitive_writer);
-criterion_main!(benches);
diff --git a/parquet/build.rs b/parquet/build.rs
deleted file mode 100644
index b42b2a4..0000000
--- a/parquet/build.rs
+++ /dev/null
@@ -1,43 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-use std::process::Command;
-
-fn main() {
- // Set Parquet version, build hash and "created by" string.
- let version = env!("CARGO_PKG_VERSION");
- let mut created_by = format!("parquet-rs version {}", version);
- if let Ok(git_hash) = run(Command::new("git").arg("rev-parse").arg("HEAD")) {
- created_by.push_str(format!(" (build {})", git_hash).as_str());
- println!("cargo:rustc-env=PARQUET_BUILD={}", git_hash);
- }
- println!("cargo:rustc-env=PARQUET_VERSION={}", version);
- println!("cargo:rustc-env=PARQUET_CREATED_BY={}", created_by);
-}
-
-/// Runs command and returns either content of stdout for successful execution,
-/// or an error message otherwise.
-fn run(command: &mut Command) -> Result<String, String> {
- println!("Running: `{:?}`", command);
- match command.output() {
- Ok(ref output) if output.status.success() => {
- Ok(String::from_utf8_lossy(&output.stdout).trim().to_string())
- }
- Ok(ref output) => Err(format!("Failed: `{:?}` ({})", command, output.status)),
- Err(error) => Err(format!("Failed: `{:?}` ({})", command, error)),
- }
-}
diff --git a/parquet/src/arrow/array_reader.rs b/parquet/src/arrow/array_reader.rs
deleted file mode 100644
index bd57cf3..0000000
--- a/parquet/src/arrow/array_reader.rs
+++ /dev/null
@@ -1,2556 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-use std::cmp::{max, min};
-use std::collections::{HashMap, HashSet};
-use std::marker::PhantomData;
-use std::mem::size_of;
-use std::result::Result::Ok;
-use std::sync::Arc;
-use std::vec::Vec;
-
-use arrow::array::{
- new_empty_array, Array, ArrayData, ArrayDataBuilder, ArrayRef, BinaryArray,
- BinaryBuilder, BooleanArray, BooleanBufferBuilder, BooleanBuilder, DecimalBuilder,
- FixedSizeBinaryArray, FixedSizeBinaryBuilder, GenericListArray, Int16BufferBuilder,
- Int32Array, Int64Array, OffsetSizeTrait, PrimitiveArray, PrimitiveBuilder,
- StringArray, StringBuilder, StructArray,
-};
-use arrow::buffer::{Buffer, MutableBuffer};
-use arrow::datatypes::{
- ArrowPrimitiveType, BooleanType as ArrowBooleanType, DataType as ArrowType,
- Date32Type as ArrowDate32Type, Date64Type as ArrowDate64Type,
- DurationMicrosecondType as ArrowDurationMicrosecondType,
- DurationMillisecondType as ArrowDurationMillisecondType,
- DurationNanosecondType as ArrowDurationNanosecondType,
- DurationSecondType as ArrowDurationSecondType, Field,
- Float32Type as ArrowFloat32Type, Float64Type as ArrowFloat64Type,
- Int16Type as ArrowInt16Type, Int32Type as ArrowInt32Type,
- Int64Type as ArrowInt64Type, Int8Type as ArrowInt8Type, IntervalUnit, Schema,
- Time32MillisecondType as ArrowTime32MillisecondType,
- Time32SecondType as ArrowTime32SecondType,
- Time64MicrosecondType as ArrowTime64MicrosecondType,
- Time64NanosecondType as ArrowTime64NanosecondType, TimeUnit as ArrowTimeUnit,
- TimestampMicrosecondType as ArrowTimestampMicrosecondType,
- TimestampMillisecondType as ArrowTimestampMillisecondType,
- TimestampNanosecondType as ArrowTimestampNanosecondType,
- TimestampSecondType as ArrowTimestampSecondType, ToByteSlice,
- UInt16Type as ArrowUInt16Type, UInt32Type as ArrowUInt32Type,
- UInt64Type as ArrowUInt64Type, UInt8Type as ArrowUInt8Type,
-};
-use arrow::util::bit_util;
-
-use crate::arrow::converter::{
- BinaryArrayConverter, BinaryConverter, Converter, DecimalArrayConverter,
- DecimalConverter, FixedLenBinaryConverter, FixedSizeArrayConverter,
- Int96ArrayConverter, Int96Converter, IntervalDayTimeArrayConverter,
- IntervalDayTimeConverter, IntervalYearMonthArrayConverter,
- IntervalYearMonthConverter, LargeBinaryArrayConverter, LargeBinaryConverter,
- LargeUtf8ArrayConverter, LargeUtf8Converter,
-};
-use crate::arrow::record_reader::RecordReader;
-use crate::arrow::schema::parquet_to_arrow_field;
-use crate::basic::{ConvertedType, Repetition, Type as PhysicalType};
-use crate::column::page::PageIterator;
-use crate::column::reader::ColumnReaderImpl;
-use crate::data_type::{
- BoolType, ByteArrayType, DataType, DoubleType, FixedLenByteArrayType, FloatType,
- Int32Type, Int64Type, Int96Type,
-};
-use crate::errors::{ParquetError, ParquetError::ArrowError, Result};
-use crate::file::reader::{FilePageIterator, FileReader};
-use crate::schema::types::{
- ColumnDescPtr, ColumnDescriptor, ColumnPath, SchemaDescPtr, Type, TypePtr,
-};
-use crate::schema::visitor::TypeVisitor;
-use std::any::Any;
-
-/// Array reader reads parquet data into arrow array.
-pub trait ArrayReader {
- fn as_any(&self) -> &dyn Any;
-
- /// Returns the arrow type of this array reader.
- fn get_data_type(&self) -> &ArrowType;
-
- /// Reads at most `batch_size` records into an arrow array and return it.
- fn next_batch(&mut self, batch_size: usize) -> Result<ArrayRef>;
-
- /// Returns the definition levels of data from last call of `next_batch`.
- /// The result is used by parent array reader to calculate its own definition
- /// levels and repetition levels, so that its parent can calculate null bitmap.
- fn get_def_levels(&self) -> Option<&[i16]>;
-
- /// Return the repetition levels of data from last call of `next_batch`.
- /// The result is used by parent array reader to calculate its own definition
- /// levels and repetition levels, so that its parent can calculate null bitmap.
- fn get_rep_levels(&self) -> Option<&[i16]>;
-}
-
-/// A NullArrayReader reads Parquet columns stored as null int32s with an Arrow
-/// NullArray type.
-pub struct NullArrayReader<T: DataType> {
- data_type: ArrowType,
- pages: Box<dyn PageIterator>,
- def_levels_buffer: Option<Buffer>,
- rep_levels_buffer: Option<Buffer>,
- column_desc: ColumnDescPtr,
- record_reader: RecordReader<T>,
- _type_marker: PhantomData<T>,
-}
-
-impl<T: DataType> NullArrayReader<T> {
- /// Construct null array reader.
- pub fn new(
- mut pages: Box<dyn PageIterator>,
- column_desc: ColumnDescPtr,
- ) -> Result<Self> {
- let mut record_reader = RecordReader::<T>::new(column_desc.clone());
- if let Some(page_reader) = pages.next() {
- record_reader.set_page_reader(page_reader?)?;
- }
-
- Ok(Self {
- data_type: ArrowType::Null,
- pages,
- def_levels_buffer: None,
- rep_levels_buffer: None,
- column_desc,
- record_reader,
- _type_marker: PhantomData,
- })
- }
-}
-
-/// Implementation of primitive array reader.
-impl<T: DataType> ArrayReader for NullArrayReader<T> {
- fn as_any(&self) -> &dyn Any {
- self
- }
-
- /// Returns data type of primitive array.
- fn get_data_type(&self) -> &ArrowType {
- &self.data_type
- }
-
- /// Reads at most `batch_size` records into array.
- fn next_batch(&mut self, batch_size: usize) -> Result<ArrayRef> {
- let mut records_read = 0usize;
- while records_read < batch_size {
- let records_to_read = batch_size - records_read;
-
- // NB can be 0 if at end of page
- let records_read_once = self.record_reader.read_records(records_to_read)?;
- records_read += records_read_once;
-
- // Record reader exhausted
- if records_read_once < records_to_read {
- if let Some(page_reader) = self.pages.next() {
- // Read from new page reader
- self.record_reader.set_page_reader(page_reader?)?;
- } else {
- // Page reader also exhausted
- break;
- }
- }
- }
-
- // convert to arrays
- let array = arrow::array::NullArray::new(records_read);
-
- // save definition and repetition buffers
- self.def_levels_buffer = self.record_reader.consume_def_levels()?;
- self.rep_levels_buffer = self.record_reader.consume_rep_levels()?;
- self.record_reader.reset();
- Ok(Arc::new(array))
- }
-
- fn get_def_levels(&self) -> Option<&[i16]> {
- self.def_levels_buffer
- .as_ref()
- .map(|buf| unsafe { buf.typed_data() })
- }
-
- fn get_rep_levels(&self) -> Option<&[i16]> {
- self.rep_levels_buffer
- .as_ref()
- .map(|buf| unsafe { buf.typed_data() })
- }
-}
-
-/// Primitive array readers are leaves of array reader tree. They accept page iterator
-/// and read them into primitive arrays.
-pub struct PrimitiveArrayReader<T: DataType> {
- data_type: ArrowType,
- pages: Box<dyn PageIterator>,
- def_levels_buffer: Option<Buffer>,
- rep_levels_buffer: Option<Buffer>,
- column_desc: ColumnDescPtr,
- record_reader: RecordReader<T>,
- _type_marker: PhantomData<T>,
-}
-
-impl<T: DataType> PrimitiveArrayReader<T> {
- /// Construct primitive array reader.
- pub fn new(
- mut pages: Box<dyn PageIterator>,
- column_desc: ColumnDescPtr,
- arrow_type: Option<ArrowType>,
- ) -> Result<Self> {
- // Check if Arrow type is specified, else create it from Parquet type
- let data_type = match arrow_type {
- Some(t) => t,
- None => parquet_to_arrow_field(column_desc.as_ref())?
- .data_type()
- .clone(),
- };
-
- let mut record_reader = RecordReader::<T>::new(column_desc.clone());
- if let Some(page_reader) = pages.next() {
- record_reader.set_page_reader(page_reader?)?;
- }
-
- Ok(Self {
- data_type,
- pages,
- def_levels_buffer: None,
- rep_levels_buffer: None,
- column_desc,
- record_reader,
- _type_marker: PhantomData,
- })
- }
-}
-
-/// Implementation of primitive array reader.
-impl<T: DataType> ArrayReader for PrimitiveArrayReader<T> {
- fn as_any(&self) -> &dyn Any {
- self
- }
-
- /// Returns data type of primitive array.
- fn get_data_type(&self) -> &ArrowType {
- &self.data_type
- }
-
- /// Reads at most `batch_size` records into array.
- fn next_batch(&mut self, batch_size: usize) -> Result<ArrayRef> {
- let mut records_read = 0usize;
- while records_read < batch_size {
- let records_to_read = batch_size - records_read;
-
- // NB can be 0 if at end of page
- let records_read_once = self.record_reader.read_records(records_to_read)?;
- records_read += records_read_once;
-
- // Record reader exhausted
- if records_read_once < records_to_read {
- if let Some(page_reader) = self.pages.next() {
- // Read from new page reader
- self.record_reader.set_page_reader(page_reader?)?;
- } else {
- // Page reader also exhausted
- break;
- }
- }
- }
-
- let target_type = self.get_data_type().clone();
- let arrow_data_type = match T::get_physical_type() {
- PhysicalType::BOOLEAN => ArrowBooleanType::DATA_TYPE,
- PhysicalType::INT32 => {
- match target_type {
- ArrowType::UInt32 => {
- // follow C++ implementation and use overflow/reinterpret cast from i32 to u32 which will map
- // `i32::MIN..0` to `(i32::MAX as u32)..u32::MAX`
- ArrowUInt32Type::DATA_TYPE
- }
- _ => ArrowInt32Type::DATA_TYPE,
- }
- }
- PhysicalType::INT64 => {
- match target_type {
- ArrowType::UInt64 => {
- // follow C++ implementation and use overflow/reinterpret cast from i64 to u64 which will map
- // `i64::MIN..0` to `(i64::MAX as u64)..u64::MAX`
- ArrowUInt64Type::DATA_TYPE
- }
- _ => ArrowInt64Type::DATA_TYPE,
- }
- }
- PhysicalType::FLOAT => ArrowFloat32Type::DATA_TYPE,
- PhysicalType::DOUBLE => ArrowFloat64Type::DATA_TYPE,
- PhysicalType::INT96
- | PhysicalType::BYTE_ARRAY
- | PhysicalType::FIXED_LEN_BYTE_ARRAY => {
- unreachable!(
- "PrimitiveArrayReaders don't support complex physical types"
- );
- }
- };
-
- // Convert to arrays by using the Parquet phyisical type.
- // The physical types are then cast to Arrow types if necessary
-
- let mut record_data = self.record_reader.consume_record_data()?;
-
- if T::get_physical_type() == PhysicalType::BOOLEAN {
- let mut boolean_buffer = BooleanBufferBuilder::new(record_data.len());
-
- for e in record_data.as_slice() {
- boolean_buffer.append(*e > 0);
- }
- record_data = boolean_buffer.finish();
- }
-
- let mut array_data = ArrayDataBuilder::new(arrow_data_type)
- .len(self.record_reader.num_values())
- .add_buffer(record_data);
-
- if let Some(b) = self.record_reader.consume_bitmap_buffer()? {
- array_data = array_data.null_bit_buffer(b);
- }
-
- let array = match T::get_physical_type() {
- PhysicalType::BOOLEAN => {
- Arc::new(BooleanArray::from(array_data.build())) as ArrayRef
- }
- PhysicalType::INT32 => {
- Arc::new(PrimitiveArray::<ArrowInt32Type>::from(array_data.build()))
- as ArrayRef
- }
- PhysicalType::INT64 => {
- Arc::new(PrimitiveArray::<ArrowInt64Type>::from(array_data.build()))
- as ArrayRef
- }
- PhysicalType::FLOAT => {
- Arc::new(PrimitiveArray::<ArrowFloat32Type>::from(array_data.build()))
- as ArrayRef
- }
- PhysicalType::DOUBLE => {
- Arc::new(PrimitiveArray::<ArrowFloat64Type>::from(array_data.build()))
- as ArrayRef
- }
- PhysicalType::INT96
- | PhysicalType::BYTE_ARRAY
- | PhysicalType::FIXED_LEN_BYTE_ARRAY => {
- unreachable!(
- "PrimitiveArrayReaders don't support complex physical types"
- );
- }
- };
-
- // cast to Arrow type
- // We make a strong assumption here that the casts should be infallible.
- // If the cast fails because of incompatible datatypes, then there might
- // be a bigger problem with how Arrow schemas are converted to Parquet.
- //
- // As there is not always a 1:1 mapping between Arrow and Parquet, there
- // are datatypes which we must convert explicitly.
- // These are:
- // - date64: we should cast int32 to date32, then date32 to date64.
- let array = match target_type {
- ArrowType::Date64 => {
- // this is cheap as it internally reinterprets the data
- let a = arrow::compute::cast(&array, &ArrowType::Date32)?;
- arrow::compute::cast(&a, &target_type)?
- }
- ArrowType::Decimal(p, s) => {
- let mut builder = DecimalBuilder::new(array.len(), p, s);
- match array.data_type() {
- ArrowType::Int32 => {
- let values = array.as_any().downcast_ref::<Int32Array>().unwrap();
- for maybe_value in values.iter() {
- match maybe_value {
- Some(value) => builder.append_value(value as i128)?,
- None => builder.append_null()?,
- }
- }
- }
- ArrowType::Int64 => {
- let values = array.as_any().downcast_ref::<Int64Array>().unwrap();
- for maybe_value in values.iter() {
- match maybe_value {
- Some(value) => builder.append_value(value as i128)?,
- None => builder.append_null()?,
- }
- }
- }
- _ => {
- return Err(ArrowError(format!(
- "Cannot convert {:?} to decimal",
- array.data_type()
- )))
- }
- }
- Arc::new(builder.finish()) as ArrayRef
- }
- _ => arrow::compute::cast(&array, &target_type)?,
- };
-
- // save definition and repetition buffers
- self.def_levels_buffer = self.record_reader.consume_def_levels()?;
- self.rep_levels_buffer = self.record_reader.consume_rep_levels()?;
- self.record_reader.reset();
- Ok(array)
- }
-
- fn get_def_levels(&self) -> Option<&[i16]> {
- self.def_levels_buffer
- .as_ref()
- .map(|buf| unsafe { buf.typed_data() })
- }
-
- fn get_rep_levels(&self) -> Option<&[i16]> {
- self.rep_levels_buffer
- .as_ref()
- .map(|buf| unsafe { buf.typed_data() })
- }
-}
-
-/// Primitive array readers are leaves of array reader tree. They accept page iterator
-/// and read them into primitive arrays.
-pub struct ComplexObjectArrayReader<T, C>
-where
- T: DataType,
- C: Converter<Vec<Option<T::T>>, ArrayRef> + 'static,
-{
- data_type: ArrowType,
- pages: Box<dyn PageIterator>,
- def_levels_buffer: Option<Vec<i16>>,
- rep_levels_buffer: Option<Vec<i16>>,
- column_desc: ColumnDescPtr,
- column_reader: Option<ColumnReaderImpl<T>>,
- converter: C,
- _parquet_type_marker: PhantomData<T>,
- _converter_marker: PhantomData<C>,
-}
-
-impl<T, C> ArrayReader for ComplexObjectArrayReader<T, C>
-where
- T: DataType,
- C: Converter<Vec<Option<T::T>>, ArrayRef> + 'static,
-{
- fn as_any(&self) -> &dyn Any {
- self
- }
-
- fn get_data_type(&self) -> &ArrowType {
- &self.data_type
- }
-
- fn next_batch(&mut self, batch_size: usize) -> Result<ArrayRef> {
- // Try to initialize column reader
- if self.column_reader.is_none() {
- self.next_column_reader()?;
- }
-
- let mut data_buffer: Vec<T::T> = Vec::with_capacity(batch_size);
- data_buffer.resize_with(batch_size, T::T::default);
-
- let mut def_levels_buffer = if self.column_desc.max_def_level() > 0 {
- let mut buf: Vec<i16> = Vec::with_capacity(batch_size);
- buf.resize_with(batch_size, || 0);
- Some(buf)
- } else {
- None
- };
-
- let mut rep_levels_buffer = if self.column_desc.max_rep_level() > 0 {
- let mut buf: Vec<i16> = Vec::with_capacity(batch_size);
- buf.resize_with(batch_size, || 0);
- Some(buf)
- } else {
- None
- };
-
- let mut num_read = 0;
-
- while self.column_reader.is_some() && num_read < batch_size {
- let num_to_read = batch_size - num_read;
- let cur_data_buf = &mut data_buffer[num_read..];
- let cur_def_levels_buf =
- def_levels_buffer.as_mut().map(|b| &mut b[num_read..]);
- let cur_rep_levels_buf =
- rep_levels_buffer.as_mut().map(|b| &mut b[num_read..]);
- let (data_read, levels_read) =
- self.column_reader.as_mut().unwrap().read_batch(
- num_to_read,
- cur_def_levels_buf,
- cur_rep_levels_buf,
- cur_data_buf,
- )?;
-
- // Fill space
- if levels_read > data_read {
- def_levels_buffer.iter().for_each(|def_levels_buffer| {
- let (mut level_pos, mut data_pos) = (levels_read, data_read);
- while level_pos > 0 && data_pos > 0 {
- if def_levels_buffer[num_read + level_pos - 1]
- == self.column_desc.max_def_level()
- {
- cur_data_buf.swap(level_pos - 1, data_pos - 1);
- level_pos -= 1;
- data_pos -= 1;
- } else {
- level_pos -= 1;
- }
- }
- });
- }
-
- let values_read = max(levels_read, data_read);
- num_read += values_read;
- // current page exhausted && page iterator exhausted
- if values_read < num_to_read && !self.next_column_reader()? {
- break;
- }
- }
-
- data_buffer.truncate(num_read);
- def_levels_buffer
- .iter_mut()
- .for_each(|buf| buf.truncate(num_read));
- rep_levels_buffer
- .iter_mut()
- .for_each(|buf| buf.truncate(num_read));
-
- self.def_levels_buffer = def_levels_buffer;
- self.rep_levels_buffer = rep_levels_buffer;
-
- let data: Vec<Option<T::T>> = if self.def_levels_buffer.is_some() {
- data_buffer
- .into_iter()
- .zip(self.def_levels_buffer.as_ref().unwrap().iter())
- .map(|(t, def_level)| {
- if *def_level == self.column_desc.max_def_level() {
- Some(t)
- } else {
- None
- }
- })
- .collect()
- } else {
- data_buffer.into_iter().map(Some).collect()
- };
-
- let mut array = self.converter.convert(data)?;
-
- if let ArrowType::Dictionary(_, _) = self.data_type {
- array = arrow::compute::cast(&array, &self.data_type)?;
- }
-
- Ok(array)
- }
-
- fn get_def_levels(&self) -> Option<&[i16]> {
- self.def_levels_buffer.as_deref()
- }
-
- fn get_rep_levels(&self) -> Option<&[i16]> {
- self.rep_levels_buffer.as_deref()
- }
-}
-
-impl<T, C> ComplexObjectArrayReader<T, C>
-where
- T: DataType,
- C: Converter<Vec<Option<T::T>>, ArrayRef> + 'static,
-{
- pub fn new(
- pages: Box<dyn PageIterator>,
- column_desc: ColumnDescPtr,
- converter: C,
- arrow_type: Option<ArrowType>,
- ) -> Result<Self> {
- let data_type = match arrow_type {
- Some(t) => t,
- None => parquet_to_arrow_field(column_desc.as_ref())?
- .data_type()
- .clone(),
- };
-
- Ok(Self {
- data_type,
- pages,
- def_levels_buffer: None,
- rep_levels_buffer: None,
- column_desc,
- column_reader: None,
- converter,
- _parquet_type_marker: PhantomData,
- _converter_marker: PhantomData,
- })
- }
-
- fn next_column_reader(&mut self) -> Result<bool> {
- Ok(match self.pages.next() {
- Some(page) => {
- self.column_reader =
- Some(ColumnReaderImpl::<T>::new(self.column_desc.clone(), page?));
- true
- }
- None => false,
- })
- }
-}
-
-/// Implementation of list array reader.
-pub struct ListArrayReader<OffsetSize: OffsetSizeTrait> {
- item_reader: Box<dyn ArrayReader>,
- data_type: ArrowType,
- item_type: ArrowType,
- list_def_level: i16,
- list_rep_level: i16,
- list_empty_def_level: i16,
- list_null_def_level: i16,
- def_level_buffer: Option<Buffer>,
- rep_level_buffer: Option<Buffer>,
- _marker: PhantomData<OffsetSize>,
-}
-
-impl<OffsetSize: OffsetSizeTrait> ListArrayReader<OffsetSize> {
- /// Construct list array reader.
- pub fn new(
- item_reader: Box<dyn ArrayReader>,
- data_type: ArrowType,
- item_type: ArrowType,
- def_level: i16,
- rep_level: i16,
- list_null_def_level: i16,
- list_empty_def_level: i16,
- ) -> Self {
- Self {
- item_reader,
- data_type,
- item_type,
- list_def_level: def_level,
- list_rep_level: rep_level,
- list_null_def_level,
- list_empty_def_level,
- def_level_buffer: None,
- rep_level_buffer: None,
- _marker: PhantomData,
- }
- }
-}
-
-macro_rules! remove_primitive_array_indices {
- ($arr: expr, $item_type:ty, $indices:expr) => {{
- let array_data = match $arr.as_any().downcast_ref::<PrimitiveArray<$item_type>>() {
- Some(a) => a,
- _ => return Err(ParquetError::General(format!("Error generating next batch for ListArray: {:?} cannot be downcast to PrimitiveArray", $arr))),
- };
- let mut builder = PrimitiveBuilder::<$item_type>::new($arr.len());
- for i in 0..array_data.len() {
- if !$indices.contains(&i) {
- if array_data.is_null(i) {
- builder.append_null()?;
- } else {
- builder.append_value(array_data.value(i))?;
- }
- }
- }
- Ok(Arc::new(builder.finish()))
- }};
-}
-
-macro_rules! remove_array_indices_custom_builder {
- ($arr: expr, $array_type:ty, $item_builder:ident, $indices:expr) => {{
- let array_data = match $arr.as_any().downcast_ref::<$array_type>() {
- Some(a) => a,
- _ => return Err(ParquetError::General(format!("Error generating next batch for ListArray: {:?} cannot be downcast to PrimitiveArray", $arr))),
- };
- let mut builder = $item_builder::new(array_data.len());
-
- for i in 0..array_data.len() {
- if !$indices.contains(&i) {
- if array_data.is_null(i) {
- builder.append_null()?;
- } else {
- builder.append_value(array_data.value(i))?;
- }
- }
- }
- Ok(Arc::new(builder.finish()))
- }};
-}
-
-macro_rules! remove_fixed_size_binary_array_indices {
- ($arr: expr, $array_type:ty, $item_builder:ident, $indices:expr, $len:expr) => {{
- let array_data = match $arr.as_any().downcast_ref::<$array_type>() {
- Some(a) => a,
- _ => return Err(ParquetError::General(format!("Error generating next batch for ListArray: {:?} cannot be downcast to PrimitiveArray", $arr))),
- };
- let mut builder = FixedSizeBinaryBuilder::new(array_data.len(), $len);
- for i in 0..array_data.len() {
- if !$indices.contains(&i) {
- if array_data.is_null(i) {
- builder.append_null()?;
- } else {
- builder.append_value(array_data.value(i))?;
- }
- }
- }
- Ok(Arc::new(builder.finish()))
- }};
-}
-
-fn remove_indices(
- arr: ArrayRef,
- item_type: ArrowType,
- indices: Vec<usize>,
-) -> Result<ArrayRef> {
- match item_type {
- ArrowType::UInt8 => remove_primitive_array_indices!(arr, ArrowUInt8Type, indices),
- ArrowType::UInt16 => {
- remove_primitive_array_indices!(arr, ArrowUInt16Type, indices)
- }
- ArrowType::UInt32 => {
- remove_primitive_array_indices!(arr, ArrowUInt32Type, indices)
- }
- ArrowType::UInt64 => {
- remove_primitive_array_indices!(arr, ArrowUInt64Type, indices)
- }
- ArrowType::Int8 => remove_primitive_array_indices!(arr, ArrowInt8Type, indices),
- ArrowType::Int16 => remove_primitive_array_indices!(arr, ArrowInt16Type, indices),
- ArrowType::Int32 => remove_primitive_array_indices!(arr, ArrowInt32Type, indices),
- ArrowType::Int64 => remove_primitive_array_indices!(arr, ArrowInt64Type, indices),
- ArrowType::Float32 => {
- remove_primitive_array_indices!(arr, ArrowFloat32Type, indices)
- }
- ArrowType::Float64 => {
- remove_primitive_array_indices!(arr, ArrowFloat64Type, indices)
- }
- ArrowType::Boolean => {
- remove_array_indices_custom_builder!(
- arr,
- BooleanArray,
- BooleanBuilder,
- indices
- )
- }
- ArrowType::Date32 => {
- remove_primitive_array_indices!(arr, ArrowDate32Type, indices)
- }
- ArrowType::Date64 => {
- remove_primitive_array_indices!(arr, ArrowDate64Type, indices)
- }
- ArrowType::Time32(ArrowTimeUnit::Second) => {
- remove_primitive_array_indices!(arr, ArrowTime32SecondType, indices)
- }
- ArrowType::Time32(ArrowTimeUnit::Millisecond) => {
- remove_primitive_array_indices!(arr, ArrowTime32MillisecondType, indices)
- }
- ArrowType::Time64(ArrowTimeUnit::Microsecond) => {
- remove_primitive_array_indices!(arr, ArrowTime64MicrosecondType, indices)
- }
- ArrowType::Time64(ArrowTimeUnit::Nanosecond) => {
- remove_primitive_array_indices!(arr, ArrowTime64NanosecondType, indices)
- }
- ArrowType::Duration(ArrowTimeUnit::Second) => {
- remove_primitive_array_indices!(arr, ArrowDurationSecondType, indices)
- }
- ArrowType::Duration(ArrowTimeUnit::Millisecond) => {
- remove_primitive_array_indices!(arr, ArrowDurationMillisecondType, indices)
- }
- ArrowType::Duration(ArrowTimeUnit::Microsecond) => {
- remove_primitive_array_indices!(arr, ArrowDurationMicrosecondType, indices)
- }
- ArrowType::Duration(ArrowTimeUnit::Nanosecond) => {
- remove_primitive_array_indices!(arr, ArrowDurationNanosecondType, indices)
- }
- ArrowType::Timestamp(ArrowTimeUnit::Second, _) => {
- remove_primitive_array_indices!(arr, ArrowTimestampSecondType, indices)
- }
- ArrowType::Timestamp(ArrowTimeUnit::Millisecond, _) => {
- remove_primitive_array_indices!(arr, ArrowTimestampMillisecondType, indices)
- }
- ArrowType::Timestamp(ArrowTimeUnit::Microsecond, _) => {
- remove_primitive_array_indices!(arr, ArrowTimestampMicrosecondType, indices)
- }
- ArrowType::Timestamp(ArrowTimeUnit::Nanosecond, _) => {
- remove_primitive_array_indices!(arr, ArrowTimestampNanosecondType, indices)
- }
- ArrowType::Utf8 => {
- remove_array_indices_custom_builder!(arr, StringArray, StringBuilder, indices)
- }
- ArrowType::Binary => {
- remove_array_indices_custom_builder!(arr, BinaryArray, BinaryBuilder, indices)
- }
- ArrowType::FixedSizeBinary(size) => remove_fixed_size_binary_array_indices!(
- arr,
- FixedSizeBinaryArray,
- FixedSizeBinaryBuilder,
- indices,
- size
- ),
- _ => Err(ParquetError::General(format!(
- "ListArray of type List({:?}) is not supported by array_reader",
- item_type
- ))),
- }
-}
-
-/// Implementation of ListArrayReader. Nested lists and lists of structs are not yet supported.
-impl<OffsetSize: OffsetSizeTrait> ArrayReader for ListArrayReader<OffsetSize> {
- fn as_any(&self) -> &dyn Any {
- self
- }
-
- /// Returns data type.
- /// This must be a List.
- fn get_data_type(&self) -> &ArrowType {
- &self.data_type
- }
-
- fn next_batch(&mut self, batch_size: usize) -> Result<ArrayRef> {
- let next_batch_array = self.item_reader.next_batch(batch_size)?;
- let item_type = self.item_reader.get_data_type().clone();
-
- if next_batch_array.len() == 0 {
- return Ok(new_empty_array(&self.data_type));
- }
- let def_levels = self
- .item_reader
- .get_def_levels()
- .ok_or_else(|| ArrowError("item_reader def levels are None.".to_string()))?;
- let rep_levels = self
- .item_reader
- .get_rep_levels()
- .ok_or_else(|| ArrowError("item_reader rep levels are None.".to_string()))?;
-
- if !((def_levels.len() == rep_levels.len())
- && (rep_levels.len() == next_batch_array.len()))
- {
- return Err(ArrowError(
- "Expected item_reader def_levels and rep_levels to be same length as batch".to_string(),
- ));
- }
-
- // List definitions can be encoded as 4 values:
- // - n + 0: the list slot is null
- // - n + 1: the list slot is not null, but is empty (i.e. [])
- // - n + 2: the list slot is not null, but its child is empty (i.e. [ null ])
- // - n + 3: the list slot is not null, and its child is not empty
- // Where n is the max definition level of the list's parent.
- // If a Parquet schema's only leaf is the list, then n = 0.
-
- // If the list index is at empty definition, the child slot is null
- let null_list_indices: Vec<usize> = def_levels
- .iter()
- .enumerate()
- .filter_map(|(index, def)| {
- if *def <= self.list_empty_def_level {
- Some(index)
- } else {
- None
- }
- })
- .collect();
- let batch_values = match null_list_indices.len() {
- 0 => next_batch_array.clone(),
- _ => remove_indices(next_batch_array.clone(), item_type, null_list_indices)?,
- };
-
- // first item in each list has rep_level = 0, subsequent items have rep_level = 1
- let mut offsets: Vec<OffsetSize> = Vec::new();
- let mut cur_offset = OffsetSize::zero();
- def_levels.iter().zip(rep_levels).for_each(|(d, r)| {
- if *r == 0 || d == &self.list_empty_def_level {
- offsets.push(cur_offset);
- }
- if d > &self.list_empty_def_level {
- cur_offset += OffsetSize::one();
- }
- });
- offsets.push(cur_offset);
-
- let num_bytes = bit_util::ceil(offsets.len(), 8);
- // TODO: A useful optimization is to use the null count to fill with
- // 0 or null, to reduce individual bits set in a loop.
- // To favour dense data, set every slot to true, then unset
- let mut null_buf = MutableBuffer::new(num_bytes).with_bitset(num_bytes, true);
- let null_slice = null_buf.as_slice_mut();
- let mut list_index = 0;
- for i in 0..rep_levels.len() {
- // If the level is lower than empty, then the slot is null.
- // When a list is non-nullable, its empty level = null level,
- // so this automatically factors that in.
- if rep_levels[i] == 0 && def_levels[i] < self.list_empty_def_level {
- bit_util::unset_bit(null_slice, list_index);
- }
- if rep_levels[i] == 0 {
- list_index += 1;
- }
- }
- let value_offsets = Buffer::from(&offsets.to_byte_slice());
-
- let list_data = ArrayData::builder(self.get_data_type().clone())
- .len(offsets.len() - 1)
- .add_buffer(value_offsets)
- .add_child_data(batch_values.data().clone())
- .null_bit_buffer(null_buf.into())
- .offset(next_batch_array.offset())
- .build();
-
- let result_array = GenericListArray::<OffsetSize>::from(list_data);
- Ok(Arc::new(result_array))
- }
-
- fn get_def_levels(&self) -> Option<&[i16]> {
- self.def_level_buffer
- .as_ref()
- .map(|buf| unsafe { buf.typed_data() })
- }
-
- fn get_rep_levels(&self) -> Option<&[i16]> {
- self.rep_level_buffer
- .as_ref()
- .map(|buf| unsafe { buf.typed_data() })
- }
-}
-
-/// Implementation of struct array reader.
-pub struct StructArrayReader {
- children: Vec<Box<dyn ArrayReader>>,
- data_type: ArrowType,
- struct_def_level: i16,
- struct_rep_level: i16,
- def_level_buffer: Option<Buffer>,
- rep_level_buffer: Option<Buffer>,
-}
-
-impl StructArrayReader {
- /// Construct struct array reader.
- pub fn new(
- data_type: ArrowType,
- children: Vec<Box<dyn ArrayReader>>,
- def_level: i16,
- rep_level: i16,
- ) -> Self {
- Self {
- data_type,
- children,
- struct_def_level: def_level,
- struct_rep_level: rep_level,
- def_level_buffer: None,
- rep_level_buffer: None,
- }
- }
-}
-
-impl ArrayReader for StructArrayReader {
- fn as_any(&self) -> &dyn Any {
- self
- }
-
- /// Returns data type.
- /// This must be a struct.
- fn get_data_type(&self) -> &ArrowType {
- &self.data_type
- }
-
- /// Read `batch_size` struct records.
- ///
- /// Definition levels of struct array is calculated as following:
- /// ```ignore
- /// def_levels[i] = min(child1_def_levels[i], child2_def_levels[i], ...,
- /// childn_def_levels[i]);
- /// ```
- ///
- /// Repetition levels of struct array is calculated as following:
- /// ```ignore
- /// rep_levels[i] = child1_rep_levels[i];
- /// ```
- ///
- /// The null bitmap of struct array is calculated from def_levels:
- /// ```ignore
- /// null_bitmap[i] = (def_levels[i] >= self.def_level);
- /// ```
- fn next_batch(&mut self, batch_size: usize) -> Result<ArrayRef> {
- if self.children.is_empty() {
- self.def_level_buffer = None;
- self.rep_level_buffer = None;
- return Ok(Arc::new(StructArray::from(Vec::new())));
- }
-
- let children_array = self
- .children
- .iter_mut()
- .map(|reader| reader.next_batch(batch_size))
- .try_fold(
- Vec::new(),
- |mut result, child_array| -> Result<Vec<ArrayRef>> {
- result.push(child_array?);
- Ok(result)
- },
- )?;
-
- // check that array child data has same size
- let children_array_len =
- children_array.first().map(|arr| arr.len()).ok_or_else(|| {
- general_err!("Struct array reader should have at least one child!")
- })?;
-
- let all_children_len_eq = children_array
- .iter()
- .all(|arr| arr.len() == children_array_len);
- if !all_children_len_eq {
- return Err(general_err!("Not all children array length are the same!"));
- }
-
- // calculate struct def level data
- let buffer_size = children_array_len * size_of::<i16>();
- let mut def_level_data_buffer = MutableBuffer::new(buffer_size);
- def_level_data_buffer.resize(buffer_size, 0);
-
- let def_level_data = def_level_data_buffer.typed_data_mut();
-
- def_level_data
- .iter_mut()
- .for_each(|v| *v = self.struct_def_level);
-
- for child in &self.children {
- if let Some(current_child_def_levels) = child.get_def_levels() {
- if current_child_def_levels.len() != children_array_len {
- return Err(general_err!("Child array length are not equal!"));
- } else {
- for i in 0..children_array_len {
- def_level_data[i] =
- min(def_level_data[i], current_child_def_levels[i]);
- }
- }
- }
- }
-
- // calculate bitmap for current array
- let mut bitmap_builder = BooleanBufferBuilder::new(children_array_len);
- for def_level in def_level_data {
- let not_null = *def_level >= self.struct_def_level;
- bitmap_builder.append(not_null);
- }
-
- // Now we can build array data
- let array_data = ArrayDataBuilder::new(self.data_type.clone())
- .len(children_array_len)
- .null_bit_buffer(bitmap_builder.finish())
- .child_data(
- children_array
- .iter()
- .map(|x| x.data().clone())
- .collect::<Vec<ArrayData>>(),
- )
- .build();
-
- // calculate struct rep level data, since struct doesn't add to repetition
- // levels, here we just need to keep repetition levels of first array
- // TODO: Verify that all children array reader has same repetition levels
- let rep_level_data = self
- .children
- .first()
- .ok_or_else(|| {
- general_err!("Struct array reader should have at least one child!")
- })?
- .get_rep_levels()
- .map(|data| -> Result<Buffer> {
- let mut buffer = Int16BufferBuilder::new(children_array_len);
- buffer.append_slice(data);
- Ok(buffer.finish())
- })
- .transpose()?;
-
- self.def_level_buffer = Some(def_level_data_buffer.into());
- self.rep_level_buffer = rep_level_data;
- Ok(Arc::new(StructArray::from(array_data)))
- }
-
- fn get_def_levels(&self) -> Option<&[i16]> {
- self.def_level_buffer
- .as_ref()
- .map(|buf| unsafe { buf.typed_data() })
- }
-
- fn get_rep_levels(&self) -> Option<&[i16]> {
- self.rep_level_buffer
- .as_ref()
- .map(|buf| unsafe { buf.typed_data() })
- }
-}
-
-/// Create array reader from parquet schema, column indices, and parquet file reader.
-pub fn build_array_reader<T>(
- parquet_schema: SchemaDescPtr,
- arrow_schema: Schema,
- column_indices: T,
- file_reader: Arc<dyn FileReader>,
-) -> Result<Box<dyn ArrayReader>>
-where
- T: IntoIterator<Item = usize>,
-{
- let mut leaves = HashMap::<*const Type, usize>::new();
-
- let mut filtered_root_names = HashSet::<String>::new();
-
- for c in column_indices {
- let column = parquet_schema.column(c).self_type() as *const Type;
-
- leaves.insert(column, c);
-
- let root = parquet_schema.get_column_root_ptr(c);
- filtered_root_names.insert(root.name().to_string());
- }
-
- if leaves.is_empty() {
- return Err(general_err!("Can't build array reader without columns!"));
- }
-
- // Only pass root fields that take part in the projection
- // to avoid traversal of columns that are not read.
- // TODO: also prune unread parts of the tree in child structures
- let filtered_root_fields = parquet_schema
- .root_schema()
- .get_fields()
- .iter()
- .filter(|field| filtered_root_names.contains(field.name()))
- .cloned()
- .collect::<Vec<_>>();
-
- let proj = Type::GroupType {
- basic_info: parquet_schema.root_schema().get_basic_info().clone(),
- fields: filtered_root_fields,
- };
-
- ArrayReaderBuilder::new(
- Arc::new(proj),
- Arc::new(arrow_schema),
- Arc::new(leaves),
- file_reader,
- )
- .build_array_reader()
-}
-
-/// Used to build array reader.
-struct ArrayReaderBuilder {
- root_schema: TypePtr,
- arrow_schema: Arc<Schema>,
- // Key: columns that need to be included in final array builder
- // Value: column index in schema
- columns_included: Arc<HashMap<*const Type, usize>>,
- file_reader: Arc<dyn FileReader>,
-}
-
-/// Used in type visitor.
-#[derive(Clone)]
-struct ArrayReaderBuilderContext {
- def_level: i16,
- rep_level: i16,
- path: ColumnPath,
-}
-
-impl Default for ArrayReaderBuilderContext {
- fn default() -> Self {
- Self {
- def_level: 0i16,
- rep_level: 0i16,
- path: ColumnPath::new(Vec::new()),
- }
- }
-}
-
-/// Create array reader by visiting schema.
-impl<'a> TypeVisitor<Option<Box<dyn ArrayReader>>, &'a ArrayReaderBuilderContext>
- for ArrayReaderBuilder
-{
- /// Build array reader for primitive type.
- /// Currently we don't have a list reader implementation, so repeated type is not
- /// supported yet.
- fn visit_primitive(
- &mut self,
- cur_type: TypePtr,
- context: &'a ArrayReaderBuilderContext,
- ) -> Result<Option<Box<dyn ArrayReader>>> {
- if self.is_included(cur_type.as_ref()) {
- let mut new_context = context.clone();
- new_context.path.append(vec![cur_type.name().to_string()]);
-
- match cur_type.get_basic_info().repetition() {
- Repetition::REPEATED => {
- new_context.def_level += 1;
- new_context.rep_level += 1;
- }
- Repetition::OPTIONAL => {
- new_context.def_level += 1;
- }
- _ => (),
- }
-
- let reader =
- self.build_for_primitive_type_inner(cur_type.clone(), &new_context)?;
-
- if cur_type.get_basic_info().repetition() == Repetition::REPEATED {
- Err(ArrowError(
- "Reading repeated field is not supported yet!".to_string(),
- ))
- } else {
- Ok(Some(reader))
- }
- } else {
- Ok(None)
- }
- }
-
- /// Build array reader for struct type.
- fn visit_struct(
- &mut self,
- cur_type: Arc<Type>,
- context: &'a ArrayReaderBuilderContext,
- ) -> Result<Option<Box<dyn ArrayReader>>> {
- let mut new_context = context.clone();
- new_context.path.append(vec![cur_type.name().to_string()]);
-
- if cur_type.get_basic_info().has_repetition() {
- match cur_type.get_basic_info().repetition() {
- Repetition::REPEATED => {
- new_context.def_level += 1;
- new_context.rep_level += 1;
- }
- Repetition::OPTIONAL => {
- new_context.def_level += 1;
- }
- _ => (),
- }
- }
-
- if let Some(reader) = self.build_for_struct_type_inner(&cur_type, &new_context)? {
- if cur_type.get_basic_info().has_repetition()
- && cur_type.get_basic_info().repetition() == Repetition::REPEATED
- {
- Err(ArrowError(
- "Reading repeated field is not supported yet!".to_string(),
- ))
- } else {
- Ok(Some(reader))
- }
- } else {
- Ok(None)
- }
- }
-
- /// Build array reader for map type.
- /// Currently this is not supported.
- fn visit_map(
- &mut self,
- _cur_type: Arc<Type>,
- _context: &'a ArrayReaderBuilderContext,
- ) -> Result<Option<Box<dyn ArrayReader>>> {
- Err(ArrowError(
- "Reading parquet map array into arrow is not supported yet!".to_string(),
- ))
- }
-
- /// Build array reader for list type.
- fn visit_list_with_item(
- &mut self,
- list_type: Arc<Type>,
- item_type: Arc<Type>,
- context: &'a ArrayReaderBuilderContext,
- ) -> Result<Option<Box<dyn ArrayReader>>> {
- let list_child = &list_type
- .get_fields()
- .first()
- .ok_or_else(|| ArrowError("List field must have a child.".to_string()))?;
- let mut new_context = context.clone();
-
- new_context.path.append(vec![list_type.name().to_string()]);
- // We need to know at what definition a list or its child is null
- let list_null_def = new_context.def_level;
- let mut list_empty_def = new_context.def_level;
-
- // If the list's root is nullable
- if let Repetition::OPTIONAL = list_type.get_basic_info().repetition() {
- new_context.def_level += 1;
- // current level is nullable, increment to get level for empty list slot
- list_empty_def += 1;
- }
-
- match list_child.get_basic_info().repetition() {
- Repetition::REPEATED => {
- new_context.def_level += 1;
- new_context.rep_level += 1;
- }
- Repetition::OPTIONAL => {
- new_context.def_level += 1;
- }
- _ => (),
- }
-
- let item_reader = self
- .dispatch(item_type.clone(), &new_context)
- .unwrap()
- .unwrap();
-
- let item_reader_type = item_reader.get_data_type().clone();
-
- match item_reader_type {
- ArrowType::List(_)
- | ArrowType::FixedSizeList(_, _)
- | ArrowType::Struct(_)
- | ArrowType::Dictionary(_, _) => Err(ArrowError(format!(
- "reading List({:?}) into arrow not supported yet",
- item_type
- ))),
- _ => {
- // a list is a group type with a single child. The list child's
- // name comes from the child's field name.
- let mut list_child = list_type.get_fields().first().ok_or(ArrowError(
- "List GroupType should have a field".to_string(),
- ))?;
- // if the child's name is "list" and it has a child, then use this child
- if list_child.name() == "list" && !list_child.get_fields().is_empty() {
- list_child = list_child.get_fields().first().unwrap();
- }
- let arrow_type = self
- .arrow_schema
- .field_with_name(list_type.name())
- .ok()
- .map(|f| f.data_type().to_owned())
- .unwrap_or_else(|| {
- ArrowType::List(Box::new(Field::new(
- list_child.name(),
- item_reader_type.clone(),
- list_child.is_optional(),
- )))
- });
-
- let list_array_reader: Box<dyn ArrayReader> = match arrow_type {
- ArrowType::List(_) => Box::new(ListArrayReader::<i32>::new(
- item_reader,
- arrow_type,
- item_reader_type,
- new_context.def_level,
- new_context.rep_level,
- list_null_def,
- list_empty_def,
- )),
- ArrowType::LargeList(_) => Box::new(ListArrayReader::<i64>::new(
- item_reader,
- arrow_type,
- item_reader_type,
- new_context.def_level,
- new_context.rep_level,
- list_null_def,
- list_empty_def,
- )),
-
- _ => {
- return Err(ArrowError(format!(
- "creating ListArrayReader with type {:?} should be unreachable",
- arrow_type
- )))
- }
- };
-
- Ok(Some(list_array_reader))
- }
- }
- }
-}
-
-impl<'a> ArrayReaderBuilder {
- /// Construct array reader builder.
- fn new(
- root_schema: TypePtr,
- arrow_schema: Arc<Schema>,
- columns_included: Arc<HashMap<*const Type, usize>>,
- file_reader: Arc<dyn FileReader>,
- ) -> Self {
- Self {
- root_schema,
- arrow_schema,
- columns_included,
- file_reader,
- }
- }
-
- /// Main entry point.
- fn build_array_reader(&mut self) -> Result<Box<dyn ArrayReader>> {
- let context = ArrayReaderBuilderContext::default();
-
- self.visit_struct(self.root_schema.clone(), &context)
- .and_then(|reader_opt| {
- reader_opt.ok_or_else(|| general_err!("Failed to build array reader!"))
- })
- }
-
- // Utility functions
-
- /// Check whether one column in included in this array reader builder.
- fn is_included(&self, t: &Type) -> bool {
- self.columns_included.contains_key(&(t as *const Type))
- }
-
- /// Creates primitive array reader for each primitive type.
- fn build_for_primitive_type_inner(
- &self,
- cur_type: TypePtr,
- context: &'a ArrayReaderBuilderContext,
- ) -> Result<Box<dyn ArrayReader>> {
- let column_desc = Arc::new(ColumnDescriptor::new(
- cur_type.clone(),
- context.def_level,
- context.rep_level,
- context.path.clone(),
- ));
- let page_iterator = Box::new(FilePageIterator::new(
- self.columns_included[&(cur_type.as_ref() as *const Type)],
- self.file_reader.clone(),
- )?);
-
- let arrow_type: Option<ArrowType> = self
- .get_arrow_field(&cur_type, context)
- .map(|f| f.data_type().clone());
-
- match cur_type.get_physical_type() {
- PhysicalType::BOOLEAN => Ok(Box::new(PrimitiveArrayReader::<BoolType>::new(
- page_iterator,
- column_desc,
- arrow_type,
- )?)),
- PhysicalType::INT32 => {
- if let Some(ArrowType::Null) = arrow_type {
- Ok(Box::new(NullArrayReader::<Int32Type>::new(
- page_iterator,
- column_desc,
- )?))
- } else {
- Ok(Box::new(PrimitiveArrayReader::<Int32Type>::new(
- page_iterator,
- column_desc,
- arrow_type,
- )?))
- }
- }
- PhysicalType::INT64 => Ok(Box::new(PrimitiveArrayReader::<Int64Type>::new(
- page_iterator,
- column_desc,
- arrow_type,
- )?)),
- PhysicalType::INT96 => {
- // get the optional timezone information from arrow type
- let timezone = arrow_type
- .as_ref()
- .map(|data_type| {
- if let ArrowType::Timestamp(_, tz) = data_type {
- tz.clone()
- } else {
- None
- }
- })
- .flatten();
- let converter = Int96Converter::new(Int96ArrayConverter { timezone });
- Ok(Box::new(ComplexObjectArrayReader::<
- Int96Type,
- Int96Converter,
- >::new(
- page_iterator,
- column_desc,
- converter,
- arrow_type,
- )?))
- }
- PhysicalType::FLOAT => Ok(Box::new(PrimitiveArrayReader::<FloatType>::new(
- page_iterator,
- column_desc,
- arrow_type,
- )?)),
- PhysicalType::DOUBLE => {
- Ok(Box::new(PrimitiveArrayReader::<DoubleType>::new(
- page_iterator,
- column_desc,
- arrow_type,
- )?))
- }
- PhysicalType::BYTE_ARRAY => {
- if cur_type.get_basic_info().converted_type() == ConvertedType::UTF8 {
- if let Some(ArrowType::LargeUtf8) = arrow_type {
- let converter =
- LargeUtf8Converter::new(LargeUtf8ArrayConverter {});
- Ok(Box::new(ComplexObjectArrayReader::<
- ByteArrayType,
- LargeUtf8Converter,
- >::new(
- page_iterator,
- column_desc,
- converter,
- arrow_type,
- )?))
- } else {
- use crate::arrow::arrow_array_reader::{
- ArrowArrayReader, StringArrayConverter,
- };
- let converter = StringArrayConverter::new();
- Ok(Box::new(ArrowArrayReader::try_new(
- *page_iterator,
- column_desc,
- converter,
- arrow_type,
- )?))
- }
- } else if let Some(ArrowType::LargeBinary) = arrow_type {
- let converter =
- LargeBinaryConverter::new(LargeBinaryArrayConverter {});
- Ok(Box::new(ComplexObjectArrayReader::<
- ByteArrayType,
- LargeBinaryConverter,
- >::new(
- page_iterator,
- column_desc,
- converter,
- arrow_type,
- )?))
- } else {
- let converter = BinaryConverter::new(BinaryArrayConverter {});
- Ok(Box::new(ComplexObjectArrayReader::<
- ByteArrayType,
- BinaryConverter,
- >::new(
- page_iterator,
- column_desc,
- converter,
- arrow_type,
- )?))
- }
- }
- PhysicalType::FIXED_LEN_BYTE_ARRAY
- if cur_type.get_basic_info().converted_type()
- == ConvertedType::DECIMAL =>
- {
- let converter = DecimalConverter::new(DecimalArrayConverter::new(
- cur_type.get_precision(),
- cur_type.get_scale(),
- ));
- Ok(Box::new(ComplexObjectArrayReader::<
- FixedLenByteArrayType,
- DecimalConverter,
- >::new(
- page_iterator,
- column_desc,
- converter,
- arrow_type,
- )?))
- }
- PhysicalType::FIXED_LEN_BYTE_ARRAY => {
- if cur_type.get_basic_info().converted_type() == ConvertedType::INTERVAL {
- let byte_width = match *cur_type {
- Type::PrimitiveType {
- ref type_length, ..
- } => *type_length,
- _ => {
- return Err(ArrowError(
- "Expected a physical type, not a group type".to_string(),
- ))
- }
- };
- if byte_width != 12 {
- return Err(ArrowError(format!(
- "Parquet interval type should have length of 12, found {}",
- byte_width
- )));
- }
- match arrow_type {
- Some(ArrowType::Interval(IntervalUnit::DayTime)) => {
- let converter = IntervalDayTimeConverter::new(
- IntervalDayTimeArrayConverter {},
- );
- Ok(Box::new(ComplexObjectArrayReader::<
- FixedLenByteArrayType,
- IntervalDayTimeConverter,
- >::new(
- page_iterator,
- column_desc,
- converter,
- arrow_type,
- )?))
- }
- Some(ArrowType::Interval(IntervalUnit::YearMonth)) => {
- let converter = IntervalYearMonthConverter::new(
- IntervalYearMonthArrayConverter {},
- );
- Ok(Box::new(ComplexObjectArrayReader::<
- FixedLenByteArrayType,
- IntervalYearMonthConverter,
- >::new(
- page_iterator,
- column_desc,
- converter,
- arrow_type,
- )?))
- }
- Some(t) => Err(ArrowError(format!(
- "Cannot write a Parquet interval to {:?}",
- t
- ))),
- None => {
- // we do not support an interval not matched to an Arrow type,
- // because we risk data loss as we won't know which of the 12 bytes
- // are or should be populated
- Err(ArrowError(
- "Cannot write a Parquet interval with no Arrow type specified.
- There is a risk of data loss as Arrow either supports YearMonth or
- DayTime precision. Without the Arrow type, we cannot infer the type.
- ".to_string()
- ))
- }
- }
- } else {
- let byte_width = match *cur_type {
- Type::PrimitiveType {
- ref type_length, ..
- } => *type_length,
- _ => {
- return Err(ArrowError(
- "Expected a physical type, not a group type".to_string(),
- ))
- }
- };
- let converter = FixedLenBinaryConverter::new(
- FixedSizeArrayConverter::new(byte_width),
- );
- Ok(Box::new(ComplexObjectArrayReader::<
- FixedLenByteArrayType,
- FixedLenBinaryConverter,
- >::new(
- page_iterator,
- column_desc,
- converter,
- arrow_type,
- )?))
- }
- }
- }
- }
-
- /// Constructs struct array reader without considering repetition.
- fn build_for_struct_type_inner(
- &mut self,
- cur_type: &Type,
- context: &'a ArrayReaderBuilderContext,
- ) -> Result<Option<Box<dyn ArrayReader>>> {
- let mut fields = Vec::with_capacity(cur_type.get_fields().len());
- let mut children_reader = Vec::with_capacity(cur_type.get_fields().len());
-
- for child in cur_type.get_fields() {
- let mut struct_context = context.clone();
- if let Some(child_reader) = self.dispatch(child.clone(), context)? {
- // TODO: this results in calling get_arrow_field twice, it could be reused
- // from child_reader above, by making child_reader carry its `Field`
- struct_context.path.append(vec![child.name().to_string()]);
- let field = match self.get_arrow_field(child, &struct_context) {
- Some(f) => f.clone(),
- _ => Field::new(
- child.name(),
- child_reader.get_data_type().clone(),
- child.is_optional(),
- ),
- };
- fields.push(field);
- children_reader.push(child_reader);
- }
- }
-
- if !fields.is_empty() {
- let arrow_type = ArrowType::Struct(fields);
- Ok(Some(Box::new(StructArrayReader::new(
- arrow_type,
- children_reader,
- context.def_level,
- context.rep_level,
- ))))
- } else {
- Ok(None)
- }
- }
-
- fn get_arrow_field(
- &self,
- cur_type: &Type,
- context: &'a ArrayReaderBuilderContext,
- ) -> Option<&Field> {
- let parts: Vec<&str> = context
- .path
- .parts()
- .iter()
- .map(|x| -> &str { x })
- .collect::<Vec<&str>>();
-
- // If the parts length is one it'll have the top level "schema" type. If
- // it's two then it'll be a top-level type that we can get from the arrow
- // schema directly.
- if parts.len() <= 2 {
- self.arrow_schema.field_with_name(cur_type.name()).ok()
- } else {
- // If it's greater than two then we need to traverse the type path
- // until we find the actual field we're looking for.
- let mut field: Option<&Field> = None;
-
- for (i, part) in parts.iter().enumerate().skip(1) {
- if i == 1 {
- field = self.arrow_schema.field_with_name(part).ok();
- } else if let Some(f) = field {
- if let ArrowType::Struct(fields) = f.data_type() {
- field = fields.iter().find(|f| f.name() == part)
- } else {
- field = None
- }
- } else {
- field = None
- }
- }
- field
- }
- }
-}
-
-#[cfg(test)]
-mod tests {
- use super::*;
- use crate::arrow::converter::{Utf8ArrayConverter, Utf8Converter};
- use crate::arrow::schema::parquet_to_arrow_schema;
- use crate::basic::{Encoding, Type as PhysicalType};
- use crate::column::page::{Page, PageReader};
- use crate::data_type::{ByteArray, DataType, Int32Type, Int64Type};
- use crate::errors::Result;
- use crate::file::reader::{FileReader, SerializedFileReader};
- use crate::schema::parser::parse_message_type;
- use crate::schema::types::{ColumnDescPtr, SchemaDescriptor};
- use crate::util::test_common::page_util::{
- DataPageBuilder, DataPageBuilderImpl, InMemoryPageIterator,
- };
- use crate::util::test_common::{get_test_file, make_pages};
- use arrow::array::{
- Array, ArrayRef, LargeListArray, ListArray, PrimitiveArray, StringArray,
- StructArray,
- };
- use arrow::datatypes::{
- ArrowPrimitiveType, DataType as ArrowType, Date32Type as ArrowDate32, Field,
- Int32Type as ArrowInt32, Int64Type as ArrowInt64,
- Time32MillisecondType as ArrowTime32MillisecondArray,
- Time64MicrosecondType as ArrowTime64MicrosecondArray,
- TimestampMicrosecondType as ArrowTimestampMicrosecondType,
- TimestampMillisecondType as ArrowTimestampMillisecondType,
- };
- use rand::distributions::uniform::SampleUniform;
- use rand::{thread_rng, Rng};
- use std::any::Any;
- use std::collections::VecDeque;
- use std::sync::Arc;
-
- fn make_column_chunks<T: DataType>(
- column_desc: ColumnDescPtr,
- encoding: Encoding,
- num_levels: usize,
- min_value: T::T,
- max_value: T::T,
- def_levels: &mut Vec<i16>,
- rep_levels: &mut Vec<i16>,
- values: &mut Vec<T::T>,
- page_lists: &mut Vec<Vec<Page>>,
- use_v2: bool,
- num_chunks: usize,
- ) where
- T::T: PartialOrd + SampleUniform + Copy,
- {
- for _i in 0..num_chunks {
- let mut pages = VecDeque::new();
- let mut data = Vec::new();
- let mut page_def_levels = Vec::new();
- let mut page_rep_levels = Vec::new();
-
- make_pages::<T>(
- column_desc.clone(),
- encoding,
- 1,
- num_levels,
- min_value,
- max_value,
- &mut page_def_levels,
- &mut page_rep_levels,
- &mut data,
- &mut pages,
- use_v2,
- );
-
- def_levels.append(&mut page_def_levels);
- rep_levels.append(&mut page_rep_levels);
- values.append(&mut data);
- page_lists.push(Vec::from(pages));
- }
- }
-
- #[test]
- fn test_primitive_array_reader_empty_pages() {
- // Construct column schema
- let message_type = "
- message test_schema {
- REQUIRED INT32 leaf;
- }
- ";
-
- let schema = parse_message_type(message_type)
- .map(|t| Arc::new(SchemaDescriptor::new(Arc::new(t))))
- .unwrap();
-
- let column_desc = schema.column(0);
- let page_iterator = EmptyPageIterator::new(schema);
-
- let mut array_reader = PrimitiveArrayReader::<Int32Type>::new(
- Box::new(page_iterator),
- column_desc,
- None,
- )
- .unwrap();
-
- // expect no values to be read
- let array = array_reader.next_batch(50).unwrap();
- assert!(array.is_empty());
- }
-
- #[test]
- fn test_primitive_array_reader_data() {
- // Construct column schema
- let message_type = "
- message test_schema {
- REQUIRED INT32 leaf;
- }
- ";
-
- let schema = parse_message_type(message_type)
- .map(|t| Arc::new(SchemaDescriptor::new(Arc::new(t))))
- .unwrap();
-
- let column_desc = schema.column(0);
-
- // Construct page iterator
- {
- let mut data = Vec::new();
- let mut page_lists = Vec::new();
- make_column_chunks::<Int32Type>(
- column_desc.clone(),
- Encoding::PLAIN,
- 100,
- 1,
- 200,
- &mut Vec::new(),
- &mut Vec::new(),
- &mut data,
- &mut page_lists,
- true,
- 2,
- );
- let page_iterator =
- InMemoryPageIterator::new(schema, column_desc.clone(), page_lists);
-
- let mut array_reader = PrimitiveArrayReader::<Int32Type>::new(
- Box::new(page_iterator),
- column_desc,
- None,
- )
- .unwrap();
-
- // Read first 50 values, which are all from the first column chunk
- let array = array_reader.next_batch(50).unwrap();
- let array = array
- .as_any()
- .downcast_ref::<PrimitiveArray<ArrowInt32>>()
- .unwrap();
-
- assert_eq!(
- &PrimitiveArray::<ArrowInt32>::from(data[0..50].to_vec()),
- array
- );
-
- // Read next 100 values, the first 50 ones are from the first column chunk,
- // and the last 50 ones are from the second column chunk
- let array = array_reader.next_batch(100).unwrap();
- let array = array
- .as_any()
- .downcast_ref::<PrimitiveArray<ArrowInt32>>()
- .unwrap();
-
- assert_eq!(
- &PrimitiveArray::<ArrowInt32>::from(data[50..150].to_vec()),
- array
- );
-
- // Try to read 100 values, however there are only 50 values
- let array = array_reader.next_batch(100).unwrap();
- let array = array
- .as_any()
- .downcast_ref::<PrimitiveArray<ArrowInt32>>()
- .unwrap();
-
- assert_eq!(
- &PrimitiveArray::<ArrowInt32>::from(data[150..200].to_vec()),
- array
- );
- }
- }
-
- macro_rules! test_primitive_array_reader_one_type {
- ($arrow_parquet_type:ty, $physical_type:expr, $converted_type_str:expr, $result_arrow_type:ty, $result_arrow_cast_type:ty, $result_primitive_type:ty) => {{
- let message_type = format!(
- "
- message test_schema {{
- REQUIRED {:?} leaf ({});
- }}
- ",
- $physical_type, $converted_type_str
- );
- let schema = parse_message_type(&message_type)
- .map(|t| Arc::new(SchemaDescriptor::new(Arc::new(t))))
- .unwrap();
-
- let column_desc = schema.column(0);
-
- // Construct page iterator
- {
- let mut data = Vec::new();
- let mut page_lists = Vec::new();
- make_column_chunks::<$arrow_parquet_type>(
- column_desc.clone(),
- Encoding::PLAIN,
- 100,
- 1,
- 200,
- &mut Vec::new(),
- &mut Vec::new(),
- &mut data,
- &mut page_lists,
- true,
- 2,
- );
- let page_iterator = InMemoryPageIterator::new(
- schema.clone(),
- column_desc.clone(),
- page_lists,
- );
- let mut array_reader = PrimitiveArrayReader::<$arrow_parquet_type>::new(
- Box::new(page_iterator),
- column_desc.clone(),
- None,
- )
- .expect("Unable to get array reader");
-
- let array = array_reader
- .next_batch(50)
- .expect("Unable to get batch from reader");
-
- let result_data_type = <$result_arrow_type>::DATA_TYPE;
- let array = array
- .as_any()
- .downcast_ref::<PrimitiveArray<$result_arrow_type>>()
- .expect(
- format!(
- "Unable to downcast {:?} to {:?}",
- array.data_type(),
- result_data_type
- )
- .as_str(),
- );
-
- // create expected array as primitive, and cast to result type
- let expected = PrimitiveArray::<$result_arrow_cast_type>::from(
- data[0..50]
- .iter()
- .map(|x| *x as $result_primitive_type)
- .collect::<Vec<$result_primitive_type>>(),
- );
- let expected = Arc::new(expected) as ArrayRef;
- let expected = arrow::compute::cast(&expected, &result_data_type)
- .expect("Unable to cast expected array");
- assert_eq!(expected.data_type(), &result_data_type);
- let expected = expected
- .as_any()
- .downcast_ref::<PrimitiveArray<$result_arrow_type>>()
- .expect(
- format!(
- "Unable to downcast expected {:?} to {:?}",
- expected.data_type(),
- result_data_type
- )
- .as_str(),
- );
- assert_eq!(expected, array);
- }
- }};
- }
-
- #[test]
- fn test_primitive_array_reader_temporal_types() {
- test_primitive_array_reader_one_type!(
- Int32Type,
- PhysicalType::INT32,
- "DATE",
- ArrowDate32,
- ArrowInt32,
- i32
- );
- test_primitive_array_reader_one_type!(
- Int32Type,
- PhysicalType::INT32,
- "TIME_MILLIS",
- ArrowTime32MillisecondArray,
- ArrowInt32,
- i32
- );
- test_primitive_array_reader_one_type!(
- Int64Type,
- PhysicalType::INT64,
- "TIME_MICROS",
- ArrowTime64MicrosecondArray,
- ArrowInt64,
- i64
- );
- test_primitive_array_reader_one_type!(
- Int64Type,
- PhysicalType::INT64,
- "TIMESTAMP_MILLIS",
- ArrowTimestampMillisecondType,
- ArrowInt64,
- i64
- );
- test_primitive_array_reader_one_type!(
- Int64Type,
- PhysicalType::INT64,
- "TIMESTAMP_MICROS",
- ArrowTimestampMicrosecondType,
- ArrowInt64,
- i64
- );
- }
-
- #[test]
- fn test_primitive_array_reader_def_and_rep_levels() {
- // Construct column schema
- let message_type = "
- message test_schema {
- REPEATED Group test_mid {
- OPTIONAL INT32 leaf;
- }
- }
- ";
-
- let schema = parse_message_type(message_type)
- .map(|t| Arc::new(SchemaDescriptor::new(Arc::new(t))))
- .unwrap();
-
- let column_desc = schema.column(0);
-
- // Construct page iterator
- {
- let mut def_levels = Vec::new();
- let mut rep_levels = Vec::new();
- let mut page_lists = Vec::new();
- make_column_chunks::<Int32Type>(
- column_desc.clone(),
- Encoding::PLAIN,
- 100,
- 1,
- 200,
- &mut def_levels,
- &mut rep_levels,
- &mut Vec::new(),
- &mut page_lists,
- true,
- 2,
- );
-
- let page_iterator =
- InMemoryPageIterator::new(schema, column_desc.clone(), page_lists);
-
- let mut array_reader = PrimitiveArrayReader::<Int32Type>::new(
- Box::new(page_iterator),
- column_desc,
- None,
- )
- .unwrap();
-
- let mut accu_len: usize = 0;
-
- // Read first 50 values, which are all from the first column chunk
- let array = array_reader.next_batch(50).unwrap();
- assert_eq!(
- Some(&def_levels[accu_len..(accu_len + array.len())]),
- array_reader.get_def_levels()
- );
- assert_eq!(
- Some(&rep_levels[accu_len..(accu_len + array.len())]),
- array_reader.get_rep_levels()
- );
- accu_len += array.len();
-
- // Read next 100 values, the first 50 ones are from the first column chunk,
- // and the last 50 ones are from the second column chunk
- let array = array_reader.next_batch(100).unwrap();
- assert_eq!(
- Some(&def_levels[accu_len..(accu_len + array.len())]),
- array_reader.get_def_levels()
- );
- assert_eq!(
- Some(&rep_levels[accu_len..(accu_len + array.len())]),
- array_reader.get_rep_levels()
- );
- accu_len += array.len();
-
- // Try to read 100 values, however there are only 50 values
- let array = array_reader.next_batch(100).unwrap();
- assert_eq!(
- Some(&def_levels[accu_len..(accu_len + array.len())]),
- array_reader.get_def_levels()
- );
- assert_eq!(
- Some(&rep_levels[accu_len..(accu_len + array.len())]),
- array_reader.get_rep_levels()
- );
- }
- }
-
- #[test]
- fn test_complex_array_reader_no_pages() {
- let message_type = "
- message test_schema {
- REPEATED Group test_mid {
- OPTIONAL BYTE_ARRAY leaf (UTF8);
- }
- }
- ";
- let schema = parse_message_type(message_type)
- .map(|t| Arc::new(SchemaDescriptor::new(Arc::new(t))))
- .unwrap();
- let column_desc = schema.column(0);
- let pages: Vec<Vec<Page>> = Vec::new();
- let page_iterator = InMemoryPageIterator::new(schema, column_desc.clone(), pages);
-
- let converter = Utf8Converter::new(Utf8ArrayConverter {});
- let mut array_reader =
- ComplexObjectArrayReader::<ByteArrayType, Utf8Converter>::new(
- Box::new(page_iterator),
- column_desc,
- converter,
- None,
- )
- .unwrap();
-
- let values_per_page = 100; // this value is arbitrary in this test - the result should always be an array of 0 length
- let array = array_reader.next_batch(values_per_page).unwrap();
- assert_eq!(array.len(), 0);
- }
-
- #[test]
- fn test_complex_array_reader_def_and_rep_levels() {
- // Construct column schema
- let message_type = "
- message test_schema {
- REPEATED Group test_mid {
- OPTIONAL BYTE_ARRAY leaf (UTF8);
- }
- }
- ";
- let num_pages = 2;
- let values_per_page = 100;
- let str_base = "Hello World";
-
- let schema = parse_message_type(message_type)
- .map(|t| Arc::new(SchemaDescriptor::new(Arc::new(t))))
- .unwrap();
-
- let max_def_level = schema.column(0).max_def_level();
- let max_rep_level = schema.column(0).max_rep_level();
-
- assert_eq!(max_def_level, 2);
- assert_eq!(max_rep_level, 1);
-
- let mut rng = thread_rng();
- let column_desc = schema.column(0);
- let mut pages: Vec<Vec<Page>> = Vec::new();
-
- let mut rep_levels = Vec::with_capacity(num_pages * values_per_page);
- let mut def_levels = Vec::with_capacity(num_pages * values_per_page);
- let mut all_values = Vec::with_capacity(num_pages * values_per_page);
-
- for i in 0..num_pages {
- let mut values = Vec::with_capacity(values_per_page);
-
- for _ in 0..values_per_page {
- let def_level = rng.gen_range(0..max_def_level + 1);
- let rep_level = rng.gen_range(0..max_rep_level + 1);
- if def_level == max_def_level {
- let len = rng.gen_range(1..str_base.len());
- let slice = &str_base[..len];
- values.push(ByteArray::from(slice));
- all_values.push(Some(slice.to_string()));
- } else {
- all_values.push(None)
- }
- rep_levels.push(rep_level);
- def_levels.push(def_level)
- }
-
- let range = i * values_per_page..(i + 1) * values_per_page;
- let mut pb =
- DataPageBuilderImpl::new(column_desc.clone(), values.len() as u32, true);
-
- pb.add_rep_levels(max_rep_level, &rep_levels.as_slice()[range.clone()]);
- pb.add_def_levels(max_def_level, &def_levels.as_slice()[range]);
- pb.add_values::<ByteArrayType>(Encoding::PLAIN, values.as_slice());
-
- let data_page = pb.consume();
- pages.push(vec![data_page]);
- }
-
- let page_iterator = InMemoryPageIterator::new(schema, column_desc.clone(), pages);
-
- let converter = Utf8Converter::new(Utf8ArrayConverter {});
- let mut array_reader =
- ComplexObjectArrayReader::<ByteArrayType, Utf8Converter>::new(
- Box::new(page_iterator),
- column_desc,
- converter,
- None,
- )
- .unwrap();
-
- let mut accu_len: usize = 0;
-
- let array = array_reader.next_batch(values_per_page / 2).unwrap();
- assert_eq!(array.len(), values_per_page / 2);
- assert_eq!(
- Some(&def_levels[accu_len..(accu_len + array.len())]),
- array_reader.get_def_levels()
- );
- assert_eq!(
- Some(&rep_levels[accu_len..(accu_len + array.len())]),
- array_reader.get_rep_levels()
- );
- accu_len += array.len();
-
- // Read next values_per_page values, the first values_per_page/2 ones are from the first column chunk,
- // and the last values_per_page/2 ones are from the second column chunk
- let array = array_reader.next_batch(values_per_page).unwrap();
- assert_eq!(array.len(), values_per_page);
- assert_eq!(
- Some(&def_levels[accu_len..(accu_len + array.len())]),
- array_reader.get_def_levels()
- );
- assert_eq!(
- Some(&rep_levels[accu_len..(accu_len + array.len())]),
- array_reader.get_rep_levels()
- );
- let strings = array.as_any().downcast_ref::<StringArray>().unwrap();
- for i in 0..array.len() {
- if array.is_valid(i) {
- assert_eq!(
- all_values[i + accu_len].as_ref().unwrap().as_str(),
- strings.value(i)
- )
- } else {
- assert_eq!(all_values[i + accu_len], None)
- }
- }
- accu_len += array.len();
-
- // Try to read values_per_page values, however there are only values_per_page/2 values
- let array = array_reader.next_batch(values_per_page).unwrap();
- assert_eq!(array.len(), values_per_page / 2);
- assert_eq!(
- Some(&def_levels[accu_len..(accu_len + array.len())]),
- array_reader.get_def_levels()
- );
- assert_eq!(
- Some(&rep_levels[accu_len..(accu_len + array.len())]),
- array_reader.get_rep_levels()
- );
- }
-
- /// Array reader for test.
- struct InMemoryArrayReader {
- data_type: ArrowType,
- array: ArrayRef,
- def_levels: Option<Vec<i16>>,
- rep_levels: Option<Vec<i16>>,
- }
-
- impl InMemoryArrayReader {
- pub fn new(
- data_type: ArrowType,
- array: ArrayRef,
- def_levels: Option<Vec<i16>>,
- rep_levels: Option<Vec<i16>>,
- ) -> Self {
- Self {
- data_type,
- array,
- def_levels,
- rep_levels,
- }
- }
- }
-
- impl ArrayReader for InMemoryArrayReader {
- fn as_any(&self) -> &dyn Any {
- self
- }
-
- fn get_data_type(&self) -> &ArrowType {
- &self.data_type
- }
-
- fn next_batch(&mut self, _batch_size: usize) -> Result<ArrayRef> {
- Ok(self.array.clone())
- }
-
- fn get_def_levels(&self) -> Option<&[i16]> {
- self.def_levels.as_deref()
- }
-
- fn get_rep_levels(&self) -> Option<&[i16]> {
- self.rep_levels.as_deref()
- }
- }
-
- /// Iterator for testing reading empty columns
- struct EmptyPageIterator {
- schema: SchemaDescPtr,
- }
-
- impl EmptyPageIterator {
- fn new(schema: SchemaDescPtr) -> Self {
- EmptyPageIterator { schema }
- }
- }
-
- impl Iterator for EmptyPageIterator {
- type Item = Result<Box<dyn PageReader>>;
-
- fn next(&mut self) -> Option<Self::Item> {
- None
- }
- }
-
- impl PageIterator for EmptyPageIterator {
- fn schema(&mut self) -> Result<SchemaDescPtr> {
- Ok(self.schema.clone())
- }
-
- fn column_schema(&mut self) -> Result<ColumnDescPtr> {
- Ok(self.schema.column(0))
- }
- }
-
- #[test]
- fn test_struct_array_reader() {
- let array_1 = Arc::new(PrimitiveArray::<ArrowInt32>::from(vec![1, 2, 3, 4, 5]));
- let array_reader_1 = InMemoryArrayReader::new(
- ArrowType::Int32,
- array_1.clone(),
- Some(vec![0, 1, 2, 3, 1]),
- Some(vec![1, 1, 1, 1, 1]),
- );
-
- let array_2 = Arc::new(PrimitiveArray::<ArrowInt32>::from(vec![5, 4, 3, 2, 1]));
- let array_reader_2 = InMemoryArrayReader::new(
- ArrowType::Int32,
- array_2.clone(),
- Some(vec![0, 1, 3, 1, 2]),
- Some(vec![1, 1, 1, 1, 1]),
- );
-
- let struct_type = ArrowType::Struct(vec![
- Field::new("f1", array_1.data_type().clone(), true),
- Field::new("f2", array_2.data_type().clone(), true),
- ]);
-
- let mut struct_array_reader = StructArrayReader::new(
- struct_type,
- vec![Box::new(array_reader_1), Box::new(array_reader_2)],
- 1,
- 1,
- );
-
- let struct_array = struct_array_reader.next_batch(5).unwrap();
- let struct_array = struct_array.as_any().downcast_ref::<StructArray>().unwrap();
-
- assert_eq!(5, struct_array.len());
- assert_eq!(
- vec![true, false, false, false, false],
- (0..5)
- .map(|idx| struct_array.data_ref().is_null(idx))
- .collect::<Vec<bool>>()
- );
- assert_eq!(
- Some(vec![0, 1, 1, 1, 1].as_slice()),
- struct_array_reader.get_def_levels()
- );
- assert_eq!(
- Some(vec![1, 1, 1, 1, 1].as_slice()),
- struct_array_reader.get_rep_levels()
- );
- }
-
- #[test]
- fn test_create_array_reader() {
- let file = get_test_file("nulls.snappy.parquet");
- let file_reader = Arc::new(SerializedFileReader::new(file).unwrap());
-
- let file_metadata = file_reader.metadata().file_metadata();
- let arrow_schema = parquet_to_arrow_schema(
- file_metadata.schema_descr(),
- file_metadata.key_value_metadata(),
- )
- .unwrap();
-
- let array_reader = build_array_reader(
- file_reader.metadata().file_metadata().schema_descr_ptr(),
- arrow_schema,
- vec![0usize].into_iter(),
- file_reader,
- )
- .unwrap();
-
- // Create arrow types
- let arrow_type = ArrowType::Struct(vec![Field::new(
- "b_struct",
- ArrowType::Struct(vec![Field::new("b_c_int", ArrowType::Int32, true)]),
- true,
- )]);
-
- assert_eq!(array_reader.get_data_type(), &arrow_type);
- }
-
- #[test]
- fn test_list_array_reader() {
- // [[1, null, 2], null, [3, 4]]
- let array = Arc::new(PrimitiveArray::<ArrowInt32>::from(vec![
- Some(1),
- None,
- Some(2),
- None,
- Some(3),
- Some(4),
- ]));
- let item_array_reader = InMemoryArrayReader::new(
- ArrowType::Int32,
- array,
- Some(vec![3, 2, 3, 0, 3, 3]),
- Some(vec![0, 1, 1, 0, 0, 1]),
- );
-
- let mut list_array_reader = ListArrayReader::<i32>::new(
- Box::new(item_array_reader),
- ArrowType::List(Box::new(Field::new("item", ArrowType::Int32, true))),
- ArrowType::Int32,
- 1,
- 1,
- 0,
- 1,
- );
-
- let next_batch = list_array_reader.next_batch(1024).unwrap();
- let list_array = next_batch.as_any().downcast_ref::<ListArray>().unwrap();
-
- assert_eq!(3, list_array.len());
- // This passes as I expect
- assert_eq!(1, list_array.null_count());
-
- assert_eq!(
- list_array
- .value(0)
- .as_any()
- .downcast_ref::<PrimitiveArray<ArrowInt32>>()
- .unwrap(),
- &PrimitiveArray::<ArrowInt32>::from(vec![Some(1), None, Some(2)])
- );
-
- assert!(list_array.is_null(1));
-
- assert_eq!(
- list_array
- .value(2)
- .as_any()
- .downcast_ref::<PrimitiveArray<ArrowInt32>>()
- .unwrap(),
- &PrimitiveArray::<ArrowInt32>::from(vec![Some(3), Some(4)])
- );
- }
-
- #[test]
- fn test_large_list_array_reader() {
- // [[1, null, 2], null, [3, 4]]
- let array = Arc::new(PrimitiveArray::<ArrowInt32>::from(vec![
- Some(1),
- None,
- Some(2),
- None,
- Some(3),
- Some(4),
- ]));
- let item_array_reader = InMemoryArrayReader::new(
- ArrowType::Int32,
- array,
- Some(vec![3, 2, 3, 0, 3, 3]),
- Some(vec![0, 1, 1, 0, 0, 1]),
- );
-
- let mut list_array_reader = ListArrayReader::<i64>::new(
- Box::new(item_array_reader),
- ArrowType::LargeList(Box::new(Field::new("item", ArrowType::Int32, true))),
- ArrowType::Int32,
- 1,
- 1,
- 0,
- 1,
- );
-
- let next_batch = list_array_reader.next_batch(1024).unwrap();
- let list_array = next_batch
- .as_any()
- .downcast_ref::<LargeListArray>()
- .unwrap();
-
- assert_eq!(3, list_array.len());
-
- assert_eq!(
- list_array
- .value(0)
- .as_any()
- .downcast_ref::<PrimitiveArray<ArrowInt32>>()
- .unwrap(),
- &PrimitiveArray::<ArrowInt32>::from(vec![Some(1), None, Some(2)])
- );
-
- assert!(list_array.is_null(1));
-
- assert_eq!(
- list_array
- .value(2)
- .as_any()
- .downcast_ref::<PrimitiveArray<ArrowInt32>>()
- .unwrap(),
- &PrimitiveArray::<ArrowInt32>::from(vec![Some(3), Some(4)])
- );
- }
-}
diff --git a/parquet/src/arrow/arrow_array_reader.rs b/parquet/src/arrow/arrow_array_reader.rs
deleted file mode 100644
index c06d872..0000000
--- a/parquet/src/arrow/arrow_array_reader.rs
+++ /dev/null
@@ -1,1562 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-use super::array_reader::ArrayReader;
-use crate::arrow::schema::parquet_to_arrow_field;
-use crate::basic::Encoding;
-use crate::errors::{ParquetError, Result};
-use crate::{
- column::page::{Page, PageIterator},
- memory::ByteBufferPtr,
- schema::types::{ColumnDescPtr, ColumnDescriptor},
-};
-use arrow::{
- array::{ArrayRef, Int16Array},
- buffer::MutableBuffer,
- datatypes::{DataType as ArrowType, ToByteSlice},
-};
-use std::{any::Any, collections::VecDeque, marker::PhantomData};
-use std::{cell::RefCell, rc::Rc};
-
-struct UnzipIter<Source, Target, State> {
- shared_state: Rc<RefCell<State>>,
- select_item_buffer: fn(&mut State) -> &mut VecDeque<Target>,
- consume_source_item: fn(source_item: Source, state: &mut State) -> Target,
-}
-
-impl<Source, Target, State> UnzipIter<Source, Target, State> {
- fn new(
- shared_state: Rc<RefCell<State>>,
- item_buffer_selector: fn(&mut State) -> &mut VecDeque<Target>,
- source_item_consumer: fn(source_item: Source, state: &mut State) -> Target,
- ) -> Self {
- Self {
- shared_state,
- select_item_buffer: item_buffer_selector,
- consume_source_item: source_item_consumer,
- }
- }
-}
-
-trait UnzipIterState<T> {
- type SourceIter: Iterator<Item = T>;
- fn source_iter(&mut self) -> &mut Self::SourceIter;
-}
-
-impl<Source, Target, State: UnzipIterState<Source>> Iterator
- for UnzipIter<Source, Target, State>
-{
- type Item = Target;
-
- fn next(&mut self) -> Option<Self::Item> {
- let mut inner = self.shared_state.borrow_mut();
- // try to get one from the stored data
- (self.select_item_buffer)(&mut *inner)
- .pop_front()
- .or_else(||
- // nothing stored, we need a new element.
- inner.source_iter().next().map(|s| {
- (self.consume_source_item)(s, &mut inner)
- }))
- }
-}
-
-struct PageBufferUnzipIterState<V, L, It> {
- iter: It,
- value_iter_buffer: VecDeque<V>,
- def_level_iter_buffer: VecDeque<L>,
- rep_level_iter_buffer: VecDeque<L>,
-}
-
-impl<V, L, It: Iterator<Item = (V, L, L)>> UnzipIterState<(V, L, L)>
- for PageBufferUnzipIterState<V, L, It>
-{
- type SourceIter = It;
-
- #[inline]
- fn source_iter(&mut self) -> &mut Self::SourceIter {
- &mut self.iter
- }
-}
-
-type ValueUnzipIter<V, L, It> =
- UnzipIter<(V, L, L), V, PageBufferUnzipIterState<V, L, It>>;
-type LevelUnzipIter<V, L, It> =
- UnzipIter<(V, L, L), L, PageBufferUnzipIterState<V, L, It>>;
-type PageUnzipResult<V, L, It> = (
- ValueUnzipIter<V, L, It>,
- LevelUnzipIter<V, L, It>,
- LevelUnzipIter<V, L, It>,
-);
-
-fn unzip_iter<V, L, It: Iterator<Item = (V, L, L)>>(it: It) -> PageUnzipResult<V, L, It> {
- let shared_data = Rc::new(RefCell::new(PageBufferUnzipIterState {
- iter: it,
- value_iter_buffer: VecDeque::new(),
- def_level_iter_buffer: VecDeque::new(),
- rep_level_iter_buffer: VecDeque::new(),
- }));
-
- let value_iter = UnzipIter::new(
- shared_data.clone(),
- |state| &mut state.value_iter_buffer,
- |(v, d, r), state| {
- state.def_level_iter_buffer.push_back(d);
- state.rep_level_iter_buffer.push_back(r);
- v
- },
- );
-
- let def_level_iter = UnzipIter::new(
- shared_data.clone(),
- |state| &mut state.def_level_iter_buffer,
- |(v, d, r), state| {
- state.value_iter_buffer.push_back(v);
- state.rep_level_iter_buffer.push_back(r);
- d
- },
- );
-
- let rep_level_iter = UnzipIter::new(
- shared_data,
- |state| &mut state.rep_level_iter_buffer,
- |(v, d, r), state| {
- state.value_iter_buffer.push_back(v);
- state.def_level_iter_buffer.push_back(d);
- r
- },
- );
-
- (value_iter, def_level_iter, rep_level_iter)
-}
-
-pub trait ArrayConverter {
- fn convert_value_bytes(
- &self,
- value_decoder: &mut impl ValueDecoder,
- num_values: usize,
- ) -> Result<arrow::array::ArrayData>;
-}
-
-pub struct ArrowArrayReader<'a, C: ArrayConverter + 'a> {
- column_desc: ColumnDescPtr,
- data_type: ArrowType,
- def_level_decoder: Box<dyn ValueDecoder + 'a>,
- rep_level_decoder: Box<dyn ValueDecoder + 'a>,
- value_decoder: Box<dyn ValueDecoder + 'a>,
- last_def_levels: Option<Int16Array>,
- last_rep_levels: Option<Int16Array>,
- array_converter: C,
-}
-
-pub(crate) struct ColumnChunkContext {
- dictionary_values: Option<Vec<ByteBufferPtr>>,
-}
-
-impl ColumnChunkContext {
- fn new() -> Self {
- Self {
- dictionary_values: None,
- }
- }
-
- fn set_dictionary(&mut self, dictionary_values: Vec<ByteBufferPtr>) {
- self.dictionary_values = Some(dictionary_values);
- }
-}
-
-type PageDecoderTuple = (
- Box<dyn ValueDecoder>,
- Box<dyn ValueDecoder>,
- Box<dyn ValueDecoder>,
-);
-
-impl<'a, C: ArrayConverter + 'a> ArrowArrayReader<'a, C> {
- pub fn try_new<P: PageIterator + 'a>(
- column_chunk_iterator: P,
- column_desc: ColumnDescPtr,
- array_converter: C,
- arrow_type: Option<ArrowType>,
- ) -> Result<Self> {
- let data_type = match arrow_type {
- Some(t) => t,
- None => parquet_to_arrow_field(column_desc.as_ref())?
- .data_type()
- .clone(),
- };
- type PageIteratorItem = Result<(Page, Rc<RefCell<ColumnChunkContext>>)>;
- let page_iter = column_chunk_iterator
- // build iterator of pages across column chunks
- .flat_map(|x| -> Box<dyn Iterator<Item = PageIteratorItem>> {
- // attach column chunk context
- let context = Rc::new(RefCell::new(ColumnChunkContext::new()));
- match x {
- Ok(page_reader) => Box::new(
- page_reader.map(move |pr| pr.map(|p| (p, context.clone()))),
- ),
- // errors from reading column chunks / row groups are propagated to page level
- Err(e) => Box::new(std::iter::once(Err(e))),
- }
- });
- // capture a clone of column_desc in closure so that it can outlive current function
- let map_page_fn_factory = |column_desc: ColumnDescPtr| {
- move |x: Result<(Page, Rc<RefCell<ColumnChunkContext>>)>| {
- x.and_then(|(page, context)| {
- Self::map_page(page, context, column_desc.as_ref())
- })
- }
- };
- let map_page_fn = map_page_fn_factory(column_desc.clone());
- // map page iterator into tuple of buffer iterators for (values, def levels, rep levels)
- // errors from lower levels are surfaced through the value decoder iterator
- let decoder_iter = page_iter.map(map_page_fn).map(|x| match x {
- Ok(iter_tuple) => iter_tuple,
- // errors from reading pages are propagated to decoder iterator level
- Err(e) => Self::map_page_error(e),
- });
- // split tuple iterator into separate iterators for (values, def levels, rep levels)
- let (value_iter, def_level_iter, rep_level_iter) = unzip_iter(decoder_iter);
-
- Ok(Self {
- column_desc,
- data_type,
- def_level_decoder: Box::new(CompositeValueDecoder::new(def_level_iter)),
- rep_level_decoder: Box::new(CompositeValueDecoder::new(rep_level_iter)),
- value_decoder: Box::new(CompositeValueDecoder::new(value_iter)),
- last_def_levels: None,
- last_rep_levels: None,
- array_converter,
- })
- }
-
- #[inline]
- fn def_levels_available(column_desc: &ColumnDescriptor) -> bool {
- column_desc.max_def_level() > 0
- }
-
- #[inline]
- fn rep_levels_available(column_desc: &ColumnDescriptor) -> bool {
- column_desc.max_rep_level() > 0
- }
-
- fn map_page_error(err: ParquetError) -> PageDecoderTuple {
- (
- Box::new(<dyn ValueDecoder>::once(Err(err.clone()))),
- Box::new(<dyn ValueDecoder>::once(Err(err.clone()))),
- Box::new(<dyn ValueDecoder>::once(Err(err))),
- )
- }
-
- // Split Result<Page> into Result<(Iterator<Values>, Iterator<DefLevels>, Iterator<RepLevels>)>
- // this method could fail, e.g. if the page encoding is not supported
- fn map_page(
- page: Page,
- column_chunk_context: Rc<RefCell<ColumnChunkContext>>,
- column_desc: &ColumnDescriptor,
- ) -> Result<PageDecoderTuple> {
- use crate::encodings::levels::LevelDecoder;
- match page {
- Page::DictionaryPage {
- buf,
- num_values,
- encoding,
- ..
- } => {
- let mut column_chunk_context = column_chunk_context.borrow_mut();
- if column_chunk_context.dictionary_values.is_some() {
- return Err(general_err!(
- "Column chunk cannot have more than one dictionary"
- ));
- }
- // create plain decoder for dictionary values
- let mut dict_decoder = Self::get_dictionary_page_decoder(
- buf,
- num_values as usize,
- encoding,
- column_desc,
- )?;
- // decode and cache dictionary values
- let dictionary_values = dict_decoder.read_dictionary_values()?;
- column_chunk_context.set_dictionary(dictionary_values);
-
- // a dictionary page doesn't return any values
- Ok((
- Box::new(<dyn ValueDecoder>::empty()),
- Box::new(<dyn ValueDecoder>::empty()),
- Box::new(<dyn ValueDecoder>::empty()),
- ))
- }
- Page::DataPage {
- buf,
- num_values,
- encoding,
- def_level_encoding,
- rep_level_encoding,
- statistics: _,
- } => {
- let mut buffer_ptr = buf;
- // create rep level decoder iterator
- let rep_level_iter: Box<dyn ValueDecoder> =
- if Self::rep_levels_available(&column_desc) {
- let mut rep_decoder = LevelDecoder::v1(
- rep_level_encoding,
- column_desc.max_rep_level(),
- );
- let rep_level_byte_len =
- rep_decoder.set_data(num_values as usize, buffer_ptr.all());
- // advance buffer pointer
- buffer_ptr = buffer_ptr.start_from(rep_level_byte_len);
- Box::new(LevelValueDecoder::new(rep_decoder))
- } else {
- Box::new(<dyn ValueDecoder>::once(Err(ParquetError::General(
- "rep levels are not available".to_string(),
- ))))
- };
- // create def level decoder iterator
- let def_level_iter: Box<dyn ValueDecoder> =
- if Self::def_levels_available(&column_desc) {
- let mut def_decoder = LevelDecoder::v1(
- def_level_encoding,
- column_desc.max_def_level(),
- );
- let def_levels_byte_len =
- def_decoder.set_data(num_values as usize, buffer_ptr.all());
- // advance buffer pointer
- buffer_ptr = buffer_ptr.start_from(def_levels_byte_len);
- Box::new(LevelValueDecoder::new(def_decoder))
- } else {
- Box::new(<dyn ValueDecoder>::once(Err(ParquetError::General(
- "def levels are not available".to_string(),
- ))))
- };
- // create value decoder iterator
- let value_iter = Self::get_value_decoder(
- buffer_ptr,
- num_values as usize,
- encoding,
- column_desc,
- column_chunk_context,
- )?;
- Ok((value_iter, def_level_iter, rep_level_iter))
- }
- Page::DataPageV2 {
- buf,
- num_values,
- encoding,
- num_nulls: _,
- num_rows: _,
- def_levels_byte_len,
- rep_levels_byte_len,
- is_compressed: _,
- statistics: _,
- } => {
- let mut offset = 0;
- // create rep level decoder iterator
- let rep_level_iter: Box<dyn ValueDecoder> =
- if Self::rep_levels_available(&column_desc) {
- let rep_levels_byte_len = rep_levels_byte_len as usize;
- let mut rep_decoder =
- LevelDecoder::v2(column_desc.max_rep_level());
- rep_decoder.set_data_range(
- num_values as usize,
- &buf,
- offset,
- rep_levels_byte_len,
- );
- offset += rep_levels_byte_len;
- Box::new(LevelValueDecoder::new(rep_decoder))
- } else {
- Box::new(<dyn ValueDecoder>::once(Err(ParquetError::General(
- "rep levels are not available".to_string(),
- ))))
- };
- // create def level decoder iterator
- let def_level_iter: Box<dyn ValueDecoder> =
- if Self::def_levels_available(&column_desc) {
- let def_levels_byte_len = def_levels_byte_len as usize;
- let mut def_decoder =
- LevelDecoder::v2(column_desc.max_def_level());
- def_decoder.set_data_range(
- num_values as usize,
- &buf,
- offset,
- def_levels_byte_len,
- );
- offset += def_levels_byte_len;
- Box::new(LevelValueDecoder::new(def_decoder))
- } else {
- Box::new(<dyn ValueDecoder>::once(Err(ParquetError::General(
- "def levels are not available".to_string(),
- ))))
- };
-
- // create value decoder iterator
- let values_buffer = buf.start_from(offset);
- let value_iter = Self::get_value_decoder(
- values_buffer,
- num_values as usize,
- encoding,
- column_desc,
- column_chunk_context,
- )?;
- Ok((value_iter, def_level_iter, rep_level_iter))
- }
- }
- }
-
- fn get_dictionary_page_decoder(
- values_buffer: ByteBufferPtr,
- num_values: usize,
- mut encoding: Encoding,
- column_desc: &ColumnDescriptor,
- ) -> Result<Box<dyn DictionaryValueDecoder>> {
- if encoding == Encoding::PLAIN || encoding == Encoding::PLAIN_DICTIONARY {
- encoding = Encoding::RLE_DICTIONARY
- }
-
- if encoding == Encoding::RLE_DICTIONARY {
- Ok(
- Self::get_plain_value_decoder(values_buffer, num_values, column_desc)
- .into_dictionary_decoder(),
- )
- } else {
- Err(nyi_err!(
- "Invalid/Unsupported encoding type for dictionary: {}",
- encoding
- ))
- }
- }
-
- fn get_value_decoder(
- values_buffer: ByteBufferPtr,
- num_values: usize,
- mut encoding: Encoding,
- column_desc: &ColumnDescriptor,
- column_chunk_context: Rc<RefCell<ColumnChunkContext>>,
- ) -> Result<Box<dyn ValueDecoder>> {
- if encoding == Encoding::PLAIN_DICTIONARY {
- encoding = Encoding::RLE_DICTIONARY;
- }
-
- match encoding {
- Encoding::PLAIN => {
- Ok(
- Self::get_plain_value_decoder(values_buffer, num_values, column_desc)
- .into_value_decoder(),
- )
- }
- Encoding::RLE_DICTIONARY => {
- if column_chunk_context.borrow().dictionary_values.is_some() {
- let value_bit_len = Self::get_column_physical_bit_len(column_desc);
- let dictionary_decoder: Box<dyn ValueDecoder> = if value_bit_len == 0
- {
- Box::new(VariableLenDictionaryDecoder::new(
- column_chunk_context,
- values_buffer,
- num_values,
- ))
- } else {
- Box::new(FixedLenDictionaryDecoder::new(
- column_chunk_context,
- values_buffer,
- num_values,
- value_bit_len,
- ))
- };
- Ok(dictionary_decoder)
- } else {
- Err(general_err!("Dictionary values have not been initialized."))
- }
- }
- // Encoding::RLE => Box::new(RleValueDecoder::new()),
- // Encoding::DELTA_BINARY_PACKED => Box::new(DeltaBitPackDecoder::new()),
- // Encoding::DELTA_LENGTH_BYTE_ARRAY => Box::new(DeltaLengthByteArrayDecoder::new()),
- // Encoding::DELTA_BYTE_ARRAY => Box::new(DeltaByteArrayDecoder::new()),
- e => return Err(nyi_err!("Encoding {} is not supported", e)),
- }
- }
-
- fn get_column_physical_bit_len(column_desc: &ColumnDescriptor) -> usize {
- use crate::basic::Type as PhysicalType;
- // parquet only supports a limited number of physical types
- // later converters cast to a more specific arrow / logical type if necessary
- match column_desc.physical_type() {
- PhysicalType::BOOLEAN => 1,
- PhysicalType::INT32 | PhysicalType::FLOAT => 32,
- PhysicalType::INT64 | PhysicalType::DOUBLE => 64,
- PhysicalType::INT96 => 96,
- PhysicalType::BYTE_ARRAY => 0,
- PhysicalType::FIXED_LEN_BYTE_ARRAY => column_desc.type_length() as usize * 8,
- }
- }
-
- fn get_plain_value_decoder(
- values_buffer: ByteBufferPtr,
- num_values: usize,
- column_desc: &ColumnDescriptor,
- ) -> Box<dyn PlainValueDecoder> {
- let value_bit_len = Self::get_column_physical_bit_len(column_desc);
- if value_bit_len == 0 {
- Box::new(VariableLenPlainDecoder::new(values_buffer, num_values))
- } else {
- Box::new(FixedLenPlainDecoder::new(
- values_buffer,
- num_values,
- value_bit_len,
- ))
- }
- }
-
- fn build_level_array(
- level_decoder: &mut impl ValueDecoder,
- batch_size: usize,
- ) -> Result<Int16Array> {
- use arrow::datatypes::Int16Type;
- let level_converter = PrimitiveArrayConverter::<Int16Type>::new();
- let array_data =
- level_converter.convert_value_bytes(level_decoder, batch_size)?;
- Ok(Int16Array::from(array_data))
- }
-}
-
-impl<C: ArrayConverter> ArrayReader for ArrowArrayReader<'static, C> {
- fn as_any(&self) -> &dyn Any {
- self
- }
-
- fn get_data_type(&self) -> &ArrowType {
- &self.data_type
- }
-
- fn next_batch(&mut self, batch_size: usize) -> Result<ArrayRef> {
- if Self::rep_levels_available(&self.column_desc) {
- // read rep levels if available
- let rep_level_array =
- Self::build_level_array(&mut self.rep_level_decoder, batch_size)?;
- self.last_rep_levels = Some(rep_level_array);
- }
-
- // check if def levels are available
- let (values_to_read, null_bitmap_array) =
- if !Self::def_levels_available(&self.column_desc) {
- // if no def levels - just read (up to) batch_size values
- (batch_size, None)
- } else {
- // if def levels are available - they determine how many values will be read
- // decode def levels, return first error if any
- let def_level_array =
- Self::build_level_array(&mut self.def_level_decoder, batch_size)?;
- let def_level_count = def_level_array.len();
- // use eq_scalar to efficiently build null bitmap array from def levels
- let null_bitmap_array = arrow::compute::eq_scalar(
- &def_level_array,
- self.column_desc.max_def_level(),
- )?;
- self.last_def_levels = Some(def_level_array);
- // efficiently calculate values to read
- let values_to_read = null_bitmap_array
- .values()
- .count_set_bits_offset(0, def_level_count);
- let maybe_null_bitmap = if values_to_read != null_bitmap_array.len() {
- Some(null_bitmap_array)
- } else {
- // shortcut if no NULLs
- None
- };
- (values_to_read, maybe_null_bitmap)
- };
-
- // read a batch of values
- // converter only creates a no-null / all value array data
- let mut value_array_data = self
- .array_converter
- .convert_value_bytes(&mut self.value_decoder, values_to_read)?;
-
- if let Some(null_bitmap_array) = null_bitmap_array {
- // Only if def levels are available - insert null values efficiently using MutableArrayData.
- // This will require value bytes to be copied again, but converter requirements are reduced.
- // With a small number of NULLs, this will only be a few copies of large byte sequences.
- let actual_batch_size = null_bitmap_array.len();
- // use_nulls is false, because null_bitmap_array is already calculated and re-used
- let mut mutable = arrow::array::MutableArrayData::new(
- vec![&value_array_data],
- false,
- actual_batch_size,
- );
- // SlicesIterator slices only the true values, NULLs are inserted to fill any gaps
- arrow::compute::SlicesIterator::new(&null_bitmap_array).for_each(
- |(start, end)| {
- // the gap needs to be filled with NULLs
- if start > mutable.len() {
- let nulls_to_add = start - mutable.len();
- mutable.extend_nulls(nulls_to_add);
- }
- // fill values, adjust start and end with NULL count so far
- let nulls_added = mutable.null_count();
- mutable.extend(0, start - nulls_added, end - nulls_added);
- },
- );
- // any remaining part is NULLs
- if mutable.len() < actual_batch_size {
- let nulls_to_add = actual_batch_size - mutable.len();
- mutable.extend_nulls(nulls_to_add);
- }
-
- value_array_data = mutable
- .into_builder()
- .null_bit_buffer(null_bitmap_array.values().clone())
- .build();
- }
- let mut array = arrow::array::make_array(value_array_data);
- if array.data_type() != &self.data_type {
- // cast array to self.data_type if necessary
- array = arrow::compute::cast(&array, &self.data_type)?
- }
- Ok(array)
- }
-
- fn get_def_levels(&self) -> Option<&[i16]> {
- self.last_def_levels.as_ref().map(|x| x.values())
- }
-
- fn get_rep_levels(&self) -> Option<&[i16]> {
- self.last_rep_levels.as_ref().map(|x| x.values())
- }
-}
-
-use crate::encodings::rle::RleDecoder;
-
-pub trait ValueDecoder {
- fn read_value_bytes(
- &mut self,
- num_values: usize,
- read_bytes: &mut dyn FnMut(&[u8], usize),
- ) -> Result<usize>;
-}
-
-trait DictionaryValueDecoder {
- fn read_dictionary_values(&mut self) -> Result<Vec<ByteBufferPtr>>;
-}
-
-trait PlainValueDecoder: ValueDecoder + DictionaryValueDecoder {
- fn into_value_decoder(self: Box<Self>) -> Box<dyn ValueDecoder>;
- fn into_dictionary_decoder(self: Box<Self>) -> Box<dyn DictionaryValueDecoder>;
-}
-
-impl<T> PlainValueDecoder for T
-where
- T: ValueDecoder + DictionaryValueDecoder + 'static,
-{
- fn into_value_decoder(self: Box<T>) -> Box<dyn ValueDecoder> {
- self
- }
-
- fn into_dictionary_decoder(self: Box<T>) -> Box<dyn DictionaryValueDecoder> {
- self
- }
-}
-
-impl dyn ValueDecoder {
- fn empty() -> impl ValueDecoder {
- SingleValueDecoder::new(Ok(0))
- }
-
- fn once(value: Result<usize>) -> impl ValueDecoder {
- SingleValueDecoder::new(value)
- }
-}
-
-impl ValueDecoder for Box<dyn ValueDecoder> {
- #[inline]
- fn read_value_bytes(
- &mut self,
- num_values: usize,
- read_bytes: &mut dyn FnMut(&[u8], usize),
- ) -> Result<usize> {
- self.as_mut().read_value_bytes(num_values, read_bytes)
- }
-}
-
-struct SingleValueDecoder {
- value: Result<usize>,
-}
-
-impl SingleValueDecoder {
- fn new(value: Result<usize>) -> Self {
- Self { value }
- }
-}
-
-impl ValueDecoder for SingleValueDecoder {
- fn read_value_bytes(
- &mut self,
- _num_values: usize,
- _read_bytes: &mut dyn FnMut(&[u8], usize),
- ) -> Result<usize> {
- self.value.clone()
- }
-}
-
-struct CompositeValueDecoder<I: Iterator<Item = Box<dyn ValueDecoder>>> {
- current_decoder: Option<Box<dyn ValueDecoder>>,
- decoder_iter: I,
-}
-
-impl<I: Iterator<Item = Box<dyn ValueDecoder>>> CompositeValueDecoder<I> {
- fn new(mut decoder_iter: I) -> Self {
- let current_decoder = decoder_iter.next();
- Self {
- current_decoder,
- decoder_iter,
- }
- }
-}
-
-impl<I: Iterator<Item = Box<dyn ValueDecoder>>> ValueDecoder
- for CompositeValueDecoder<I>
-{
- fn read_value_bytes(
- &mut self,
- num_values: usize,
- read_bytes: &mut dyn FnMut(&[u8], usize),
- ) -> Result<usize> {
- let mut values_to_read = num_values;
- while values_to_read > 0 {
- let value_decoder = match self.current_decoder.as_mut() {
- Some(d) => d,
- // no more decoders
- None => break,
- };
- while values_to_read > 0 {
- let values_read =
- value_decoder.read_value_bytes(values_to_read, read_bytes)?;
- if values_read > 0 {
- values_to_read -= values_read;
- } else {
- // no more values in current decoder
- self.current_decoder = self.decoder_iter.next();
- break;
- }
- }
- }
-
- Ok(num_values - values_to_read)
- }
-}
-
-struct LevelValueDecoder {
- level_decoder: crate::encodings::levels::LevelDecoder,
- level_value_buffer: Vec<i16>,
-}
-
-impl LevelValueDecoder {
- fn new(level_decoder: crate::encodings::levels::LevelDecoder) -> Self {
- Self {
- level_decoder,
- level_value_buffer: vec![0i16; 2048],
- }
- }
-}
-
-impl ValueDecoder for LevelValueDecoder {
- fn read_value_bytes(
- &mut self,
- num_values: usize,
- read_bytes: &mut dyn FnMut(&[u8], usize),
- ) -> Result<usize> {
- let value_size = std::mem::size_of::<i16>();
- let mut total_values_read = 0;
- while total_values_read < num_values {
- let values_to_read = std::cmp::min(
- num_values - total_values_read,
- self.level_value_buffer.len(),
- );
- let values_read = match self
- .level_decoder
- .get(&mut self.level_value_buffer[..values_to_read])
- {
- Ok(values_read) => values_read,
- Err(e) => return Err(e),
- };
- if values_read > 0 {
- let level_value_bytes =
- &self.level_value_buffer.to_byte_slice()[..values_read * value_size];
- read_bytes(level_value_bytes, values_read);
- total_values_read += values_read;
- } else {
- break;
- }
- }
- Ok(total_values_read)
- }
-}
-
-pub(crate) struct FixedLenPlainDecoder {
- data: ByteBufferPtr,
- num_values: usize,
- value_bit_len: usize,
-}
-
-impl FixedLenPlainDecoder {
- pub(crate) fn new(
- data: ByteBufferPtr,
- num_values: usize,
- value_bit_len: usize,
- ) -> Self {
- Self {
- data,
- num_values,
- value_bit_len,
- }
- }
-}
-
-impl DictionaryValueDecoder for FixedLenPlainDecoder {
- fn read_dictionary_values(&mut self) -> Result<Vec<ByteBufferPtr>> {
- let value_byte_len = self.value_bit_len / 8;
- let available_values = self.data.len() / value_byte_len;
- let values_to_read = std::cmp::min(available_values, self.num_values);
- let byte_len = values_to_read * value_byte_len;
- let values = vec![self.data.range(0, byte_len)];
- self.num_values = 0;
- self.data.set_range(self.data.start(), 0);
- Ok(values)
- }
-}
-
-impl ValueDecoder for FixedLenPlainDecoder {
- fn read_value_bytes(
- &mut self,
- num_values: usize,
- read_bytes: &mut dyn FnMut(&[u8], usize),
- ) -> Result<usize> {
- let available_values = self.data.len() * 8 / self.value_bit_len;
- if available_values > 0 {
- let values_to_read = std::cmp::min(available_values, num_values);
- let byte_len = values_to_read * self.value_bit_len / 8;
- read_bytes(&self.data.data()[..byte_len], values_to_read);
- self.data
- .set_range(self.data.start() + byte_len, self.data.len() - byte_len);
- Ok(values_to_read)
- } else {
- Ok(0)
- }
- }
-}
-
-pub(crate) struct VariableLenPlainDecoder {
- data: ByteBufferPtr,
- num_values: usize,
- position: usize,
-}
-
-impl VariableLenPlainDecoder {
- pub(crate) fn new(data: ByteBufferPtr, num_values: usize) -> Self {
- Self {
- data,
- num_values,
- position: 0,
- }
- }
-}
-
-impl DictionaryValueDecoder for VariableLenPlainDecoder {
- fn read_dictionary_values(&mut self) -> Result<Vec<ByteBufferPtr>> {
- const LEN_SIZE: usize = std::mem::size_of::<u32>();
- let data = self.data.data();
- let data_len = data.len();
- let values_to_read = self.num_values;
- let mut values = Vec::with_capacity(values_to_read);
- let mut values_read = 0;
- while self.position < data_len && values_read < values_to_read {
- let len: usize =
- read_num_bytes!(u32, LEN_SIZE, data[self.position..]) as usize;
- self.position += LEN_SIZE;
- if data_len < self.position + len {
- return Err(eof_err!("Not enough bytes to decode"));
- }
- values.push(self.data.range(self.position, len));
- self.position += len;
- values_read += 1;
- }
- self.num_values -= values_read;
- Ok(values)
- }
-}
-
-impl ValueDecoder for VariableLenPlainDecoder {
- fn read_value_bytes(
- &mut self,
- num_values: usize,
- read_bytes: &mut dyn FnMut(&[u8], usize),
- ) -> Result<usize> {
- const LEN_SIZE: usize = std::mem::size_of::<u32>();
- let data = self.data.data();
- let data_len = data.len();
- let values_to_read = std::cmp::min(self.num_values, num_values);
- let mut values_read = 0;
- while self.position < data_len && values_read < values_to_read {
- let len: usize =
- read_num_bytes!(u32, LEN_SIZE, data[self.position..]) as usize;
- self.position += LEN_SIZE;
- if data_len < self.position + len {
- return Err(eof_err!("Not enough bytes to decode"));
- }
- read_bytes(&data[self.position..][..len], 1);
- self.position += len;
- values_read += 1;
- }
- self.num_values -= values_read;
- Ok(values_read)
- }
-}
-
-pub(crate) struct FixedLenDictionaryDecoder {
- context_ref: Rc<RefCell<ColumnChunkContext>>,
- key_data_bufer: ByteBufferPtr,
- num_values: usize,
- rle_decoder: RleDecoder,
- value_byte_len: usize,
- keys_buffer: Vec<i32>,
-}
-
-impl FixedLenDictionaryDecoder {
- pub(crate) fn new(
- column_chunk_context: Rc<RefCell<ColumnChunkContext>>,
- key_data_bufer: ByteBufferPtr,
- num_values: usize,
- value_bit_len: usize,
- ) -> Self {
- assert!(
- value_bit_len % 8 == 0,
- "value_bit_size must be a multiple of 8"
- );
- // First byte in `data` is bit width
- let bit_width = key_data_bufer.data()[0];
- let mut rle_decoder = RleDecoder::new(bit_width);
- rle_decoder.set_data(key_data_bufer.start_from(1));
-
- Self {
- context_ref: column_chunk_context,
- key_data_bufer,
- num_values,
- rle_decoder,
- value_byte_len: value_bit_len / 8,
- keys_buffer: vec![0; 2048],
- }
- }
-}
-
-impl ValueDecoder for FixedLenDictionaryDecoder {
- fn read_value_bytes(
- &mut self,
- num_values: usize,
- read_bytes: &mut dyn FnMut(&[u8], usize),
- ) -> Result<usize> {
- if self.num_values == 0 {
- return Ok(0);
- }
- let context = self.context_ref.borrow();
- let values = context.dictionary_values.as_ref().unwrap();
- let input_value_bytes = values[0].data();
- // read no more than available values or requested values
- let values_to_read = std::cmp::min(self.num_values, num_values);
- let mut values_read = 0;
- while values_read < values_to_read {
- // read values in batches of up to self.keys_buffer.len()
- let keys_to_read =
- std::cmp::min(values_to_read - values_read, self.keys_buffer.len());
- let keys_read = match self
- .rle_decoder
- .get_batch(&mut self.keys_buffer[..keys_to_read])
- {
- Ok(keys_read) => keys_read,
- Err(e) => return Err(e),
- };
- if keys_read == 0 {
- self.num_values = 0;
- return Ok(values_read);
- }
- for i in 0..keys_read {
- let key = self.keys_buffer[i] as usize;
- read_bytes(
- &input_value_bytes[key * self.value_byte_len..]
- [..self.value_byte_len],
- 1,
- );
- }
- values_read += keys_read;
- }
- self.num_values -= values_read;
- Ok(values_read)
- }
-}
-
-pub(crate) struct VariableLenDictionaryDecoder {
- context_ref: Rc<RefCell<ColumnChunkContext>>,
- key_data_bufer: ByteBufferPtr,
- num_values: usize,
- rle_decoder: RleDecoder,
- keys_buffer: Vec<i32>,
-}
-
-impl VariableLenDictionaryDecoder {
- pub(crate) fn new(
- column_chunk_context: Rc<RefCell<ColumnChunkContext>>,
- key_data_bufer: ByteBufferPtr,
- num_values: usize,
- ) -> Self {
- // First byte in `data` is bit width
- let bit_width = key_data_bufer.data()[0];
- let mut rle_decoder = RleDecoder::new(bit_width);
- rle_decoder.set_data(key_data_bufer.start_from(1));
-
- Self {
- context_ref: column_chunk_context,
- key_data_bufer,
- num_values,
- rle_decoder,
- keys_buffer: vec![0; 2048],
- }
- }
-}
-
-impl ValueDecoder for VariableLenDictionaryDecoder {
- fn read_value_bytes(
- &mut self,
- num_values: usize,
- read_bytes: &mut dyn FnMut(&[u8], usize),
- ) -> Result<usize> {
- if self.num_values == 0 {
- return Ok(0);
- }
- let context = self.context_ref.borrow();
- let values = context.dictionary_values.as_ref().unwrap();
- let values_to_read = std::cmp::min(self.num_values, num_values);
- let mut values_read = 0;
- while values_read < values_to_read {
- // read values in batches of up to self.keys_buffer.len()
- let keys_to_read =
- std::cmp::min(values_to_read - values_read, self.keys_buffer.len());
- let keys_read = match self
- .rle_decoder
- .get_batch(&mut self.keys_buffer[..keys_to_read])
- {
- Ok(keys_read) => keys_read,
- Err(e) => return Err(e),
- };
- if keys_read == 0 {
- self.num_values = 0;
- return Ok(values_read);
- }
- for i in 0..keys_read {
- let key = self.keys_buffer[i] as usize;
- read_bytes(values[key].data(), 1);
- }
- values_read += keys_read;
- }
- self.num_values -= values_read;
- Ok(values_read)
- }
-}
-
-use arrow::datatypes::ArrowPrimitiveType;
-
-pub struct PrimitiveArrayConverter<T: ArrowPrimitiveType> {
- _phantom_data: PhantomData<T>,
-}
-
-impl<T: ArrowPrimitiveType> PrimitiveArrayConverter<T> {
- pub fn new() -> Self {
- Self {
- _phantom_data: PhantomData,
- }
- }
-}
-
-impl<T: ArrowPrimitiveType> ArrayConverter for PrimitiveArrayConverter<T> {
- fn convert_value_bytes(
- &self,
- value_decoder: &mut impl ValueDecoder,
- num_values: usize,
- ) -> Result<arrow::array::ArrayData> {
- let value_size = T::get_byte_width();
- let values_byte_capacity = num_values * value_size;
- let mut values_buffer = MutableBuffer::new(values_byte_capacity);
-
- value_decoder.read_value_bytes(num_values, &mut |value_bytes, _| {
- values_buffer.extend_from_slice(value_bytes);
- })?;
-
- // calculate actual data_len, which may be different from the iterator's upper bound
- let value_count = values_buffer.len() / value_size;
- let array_data = arrow::array::ArrayData::builder(T::DATA_TYPE)
- .len(value_count)
- .add_buffer(values_buffer.into())
- .build();
- Ok(array_data)
- }
-}
-
-pub struct StringArrayConverter {}
-
-impl StringArrayConverter {
- pub fn new() -> Self {
- Self {}
- }
-}
-
-impl ArrayConverter for StringArrayConverter {
- fn convert_value_bytes(
- &self,
- value_decoder: &mut impl ValueDecoder,
- num_values: usize,
- ) -> Result<arrow::array::ArrayData> {
- use arrow::datatypes::ArrowNativeType;
- let offset_size = std::mem::size_of::<i32>();
- let mut offsets_buffer = MutableBuffer::new((num_values + 1) * offset_size);
- // allocate initial capacity of 1 byte for each item
- let values_byte_capacity = num_values;
- let mut values_buffer = MutableBuffer::new(values_byte_capacity);
-
- let mut length_so_far = i32::default();
- offsets_buffer.push(length_so_far);
-
- value_decoder.read_value_bytes(num_values, &mut |value_bytes, values_read| {
- debug_assert_eq!(
- values_read, 1,
- "offset length value buffers can only contain bytes for a single value"
- );
- length_so_far +=
- <i32 as ArrowNativeType>::from_usize(value_bytes.len()).unwrap();
- // this should be safe because a ValueDecoder should not read more than num_values
- unsafe {
- offsets_buffer.push_unchecked(length_so_far);
- }
- values_buffer.extend_from_slice(value_bytes);
- })?;
- // calculate actual data_len, which may be different from the iterator's upper bound
- let data_len = (offsets_buffer.len() / offset_size) - 1;
- let array_data = arrow::array::ArrayData::builder(ArrowType::Utf8)
- .len(data_len)
- .add_buffer(offsets_buffer.into())
- .add_buffer(values_buffer.into())
- .build();
- Ok(array_data)
- }
-}
-
-#[cfg(test)]
-mod tests {
- use super::*;
- use crate::column::page::Page;
- use crate::data_type::ByteArray;
- use crate::data_type::ByteArrayType;
- use crate::schema::parser::parse_message_type;
- use crate::schema::types::SchemaDescriptor;
- use crate::util::test_common::page_util::{
- DataPageBuilder, DataPageBuilderImpl, InMemoryPageIterator,
- };
- use crate::{
- basic::Encoding, column::page::PageReader, schema::types::SchemaDescPtr,
- };
- use arrow::array::{PrimitiveArray, StringArray};
- use arrow::datatypes::Int32Type as ArrowInt32;
- use rand::{distributions::uniform::SampleUniform, thread_rng, Rng};
- use std::sync::Arc;
-
- /// Iterator for testing reading empty columns
- struct EmptyPageIterator {
- schema: SchemaDescPtr,
- }
-
- impl EmptyPageIterator {
- fn new(schema: SchemaDescPtr) -> Self {
- EmptyPageIterator { schema }
- }
- }
-
- impl Iterator for EmptyPageIterator {
- type Item = Result<Box<dyn PageReader>>;
-
- fn next(&mut self) -> Option<Self::Item> {
- None
- }
- }
-
- impl PageIterator for EmptyPageIterator {
- fn schema(&mut self) -> Result<SchemaDescPtr> {
- Ok(self.schema.clone())
- }
-
- fn column_schema(&mut self) -> Result<ColumnDescPtr> {
- Ok(self.schema.column(0))
- }
- }
-
- #[test]
- fn test_array_reader_empty_pages() {
- // Construct column schema
- let message_type = "
- message test_schema {
- REQUIRED INT32 leaf;
- }
- ";
-
- let schema = parse_message_type(message_type)
- .map(|t| Arc::new(SchemaDescriptor::new(Arc::new(t))))
- .unwrap();
-
- let column_desc = schema.column(0);
- let page_iterator = EmptyPageIterator::new(schema);
-
- let converter = PrimitiveArrayConverter::<arrow::datatypes::Int32Type>::new();
- let mut array_reader =
- ArrowArrayReader::try_new(page_iterator, column_desc, converter, None)
- .unwrap();
-
- // expect no values to be read
- let array = array_reader.next_batch(50).unwrap();
- assert!(array.is_empty());
- }
-
- fn make_column_chunks<T: crate::data_type::DataType>(
- column_desc: ColumnDescPtr,
- encoding: Encoding,
- num_levels: usize,
- min_value: T::T,
- max_value: T::T,
- def_levels: &mut Vec<i16>,
- rep_levels: &mut Vec<i16>,
- values: &mut Vec<T::T>,
- page_lists: &mut Vec<Vec<Page>>,
- use_v2: bool,
- num_chunks: usize,
- ) where
- T::T: PartialOrd + SampleUniform + Copy,
- {
- for _i in 0..num_chunks {
- let mut pages = VecDeque::new();
- let mut data = Vec::new();
- let mut page_def_levels = Vec::new();
- let mut page_rep_levels = Vec::new();
-
- crate::util::test_common::make_pages::<T>(
- column_desc.clone(),
- encoding,
- 1,
- num_levels,
- min_value,
- max_value,
- &mut page_def_levels,
- &mut page_rep_levels,
- &mut data,
- &mut pages,
- use_v2,
- );
-
- def_levels.append(&mut page_def_levels);
- rep_levels.append(&mut page_rep_levels);
- values.append(&mut data);
- page_lists.push(Vec::from(pages));
- }
- }
-
- #[test]
- fn test_primitive_array_reader_data() {
- // Construct column schema
- let message_type = "
- message test_schema {
- REQUIRED INT32 leaf;
- }
- ";
-
- let schema = parse_message_type(message_type)
- .map(|t| Arc::new(SchemaDescriptor::new(Arc::new(t))))
- .unwrap();
-
- let column_desc = schema.column(0);
-
- // Construct page iterator
- {
- let mut data = Vec::new();
- let mut page_lists = Vec::new();
- make_column_chunks::<crate::data_type::Int32Type>(
- column_desc.clone(),
- Encoding::PLAIN,
- 100,
- 1,
- 200,
- &mut Vec::new(),
- &mut Vec::new(),
- &mut data,
- &mut page_lists,
- true,
- 2,
- );
- let page_iterator =
- InMemoryPageIterator::new(schema, column_desc.clone(), page_lists);
-
- let converter = PrimitiveArrayConverter::<arrow::datatypes::Int32Type>::new();
- let mut array_reader =
- ArrowArrayReader::try_new(page_iterator, column_desc, converter, None)
- .unwrap();
-
- // Read first 50 values, which are all from the first column chunk
- let array = array_reader.next_batch(50).unwrap();
- let array = array
- .as_any()
- .downcast_ref::<PrimitiveArray<ArrowInt32>>()
- .unwrap();
-
- assert_eq!(
- &PrimitiveArray::<ArrowInt32>::from(data[0..50].to_vec()),
- array
- );
-
- // Read next 100 values, the first 50 ones are from the first column chunk,
- // and the last 50 ones are from the second column chunk
- let array = array_reader.next_batch(100).unwrap();
- let array = array
- .as_any()
- .downcast_ref::<PrimitiveArray<ArrowInt32>>()
- .unwrap();
-
- assert_eq!(
- &PrimitiveArray::<ArrowInt32>::from(data[50..150].to_vec()),
- array
- );
-
- // Try to read 100 values, however there are only 50 values
- let array = array_reader.next_batch(100).unwrap();
- let array = array
- .as_any()
- .downcast_ref::<PrimitiveArray<ArrowInt32>>()
- .unwrap();
-
- assert_eq!(
- &PrimitiveArray::<ArrowInt32>::from(data[150..200].to_vec()),
- array
- );
- }
- }
-
- #[test]
- fn test_primitive_array_reader_def_and_rep_levels() {
- // Construct column schema
- let message_type = "
- message test_schema {
- REPEATED Group test_mid {
- OPTIONAL INT32 leaf;
- }
- }
- ";
-
- let schema = parse_message_type(message_type)
- .map(|t| Arc::new(SchemaDescriptor::new(Arc::new(t))))
- .unwrap();
-
- let column_desc = schema.column(0);
-
- // Construct page iterator
- {
- let mut def_levels = Vec::new();
- let mut rep_levels = Vec::new();
- let mut page_lists = Vec::new();
- make_column_chunks::<crate::data_type::Int32Type>(
- column_desc.clone(),
- Encoding::PLAIN,
- 100,
- 1,
- 200,
- &mut def_levels,
- &mut rep_levels,
- &mut Vec::new(),
- &mut page_lists,
- true,
- 2,
- );
-
- let page_iterator =
- InMemoryPageIterator::new(schema, column_desc.clone(), page_lists);
-
- let converter = PrimitiveArrayConverter::<arrow::datatypes::Int32Type>::new();
- let mut array_reader =
- ArrowArrayReader::try_new(page_iterator, column_desc, converter, None)
- .unwrap();
-
- let mut accu_len: usize = 0;
-
- // Read first 50 values, which are all from the first column chunk
- let array = array_reader.next_batch(50).unwrap();
- assert_eq!(
- Some(&def_levels[accu_len..(accu_len + array.len())]),
- array_reader.get_def_levels()
- );
- assert_eq!(
- Some(&rep_levels[accu_len..(accu_len + array.len())]),
- array_reader.get_rep_levels()
- );
- accu_len += array.len();
-
- // Read next 100 values, the first 50 ones are from the first column chunk,
- // and the last 50 ones are from the second column chunk
- let array = array_reader.next_batch(100).unwrap();
- assert_eq!(
- Some(&def_levels[accu_len..(accu_len + array.len())]),
- array_reader.get_def_levels()
- );
- assert_eq!(
- Some(&rep_levels[accu_len..(accu_len + array.len())]),
- array_reader.get_rep_levels()
- );
- accu_len += array.len();
-
- // Try to read 100 values, however there are only 50 values
- let array = array_reader.next_batch(100).unwrap();
- assert_eq!(
- Some(&def_levels[accu_len..(accu_len + array.len())]),
- array_reader.get_def_levels()
- );
- assert_eq!(
- Some(&rep_levels[accu_len..(accu_len + array.len())]),
- array_reader.get_rep_levels()
- );
-
- assert_eq!(accu_len + array.len(), 200);
- }
- }
-
- #[test]
- fn test_arrow_array_reader_string() {
- // Construct column schema
- let message_type = "
- message test_schema {
- REPEATED Group test_mid {
- OPTIONAL BYTE_ARRAY leaf (UTF8);
- }
- }
- ";
- let num_pages = 2;
- let values_per_page = 100;
- let str_base = "Hello World";
-
- let schema = parse_message_type(message_type)
- .map(|t| Arc::new(SchemaDescriptor::new(Arc::new(t))))
- .unwrap();
- let column_desc = schema.column(0);
- let max_def_level = column_desc.max_def_level();
- let max_rep_level = column_desc.max_rep_level();
-
- assert_eq!(max_def_level, 2);
- assert_eq!(max_rep_level, 1);
-
- let mut rng = thread_rng();
- let mut pages: Vec<Vec<Page>> = Vec::new();
-
- let mut rep_levels = Vec::with_capacity(num_pages * values_per_page);
- let mut def_levels = Vec::with_capacity(num_pages * values_per_page);
- let mut all_values = Vec::with_capacity(num_pages * values_per_page);
-
- for i in 0..num_pages {
- let mut values = Vec::with_capacity(values_per_page);
-
- for _ in 0..values_per_page {
- let def_level = rng.gen_range(0..max_def_level + 1);
- let rep_level = rng.gen_range(0..max_rep_level + 1);
- if def_level == max_def_level {
- let len = rng.gen_range(1..str_base.len());
- let slice = &str_base[..len];
- values.push(ByteArray::from(slice));
- all_values.push(Some(slice.to_string()));
- } else {
- all_values.push(None)
- }
- rep_levels.push(rep_level);
- def_levels.push(def_level)
- }
-
- let range = i * values_per_page..(i + 1) * values_per_page;
- let mut pb =
- DataPageBuilderImpl::new(column_desc.clone(), values.len() as u32, true);
-
- pb.add_rep_levels(max_rep_level, &rep_levels.as_slice()[range.clone()]);
- pb.add_def_levels(max_def_level, &def_levels.as_slice()[range]);
- pb.add_values::<ByteArrayType>(Encoding::PLAIN, values.as_slice());
-
- let data_page = pb.consume();
- pages.push(vec![data_page]);
- }
-
- let page_iterator = InMemoryPageIterator::new(schema, column_desc.clone(), pages);
- let converter = StringArrayConverter::new();
- let mut array_reader =
- ArrowArrayReader::try_new(page_iterator, column_desc, converter, None)
- .unwrap();
-
- let mut accu_len: usize = 0;
-
- let array = array_reader.next_batch(values_per_page / 2).unwrap();
- assert_eq!(array.len(), values_per_page / 2);
- assert_eq!(
- Some(&def_levels[accu_len..(accu_len + array.len())]),
- array_reader.get_def_levels()
- );
- assert_eq!(
- Some(&rep_levels[accu_len..(accu_len + array.len())]),
- array_reader.get_rep_levels()
- );
- accu_len += array.len();
-
- // Read next values_per_page values, the first values_per_page/2 ones are from the first column chunk,
- // and the last values_per_page/2 ones are from the second column chunk
- let array = array_reader.next_batch(values_per_page).unwrap();
- assert_eq!(array.len(), values_per_page);
- assert_eq!(
- Some(&def_levels[accu_len..(accu_len + array.len())]),
- array_reader.get_def_levels()
- );
- assert_eq!(
- Some(&rep_levels[accu_len..(accu_len + array.len())]),
- array_reader.get_rep_levels()
- );
- let strings = array.as_any().downcast_ref::<StringArray>().unwrap();
- for i in 0..array.len() {
- if array.is_valid(i) {
- assert_eq!(
- all_values[i + accu_len].as_ref().unwrap().as_str(),
- strings.value(i)
- )
- } else {
- assert_eq!(all_values[i + accu_len], None)
- }
- }
- accu_len += array.len();
-
- // Try to read values_per_page values, however there are only values_per_page/2 values
- let array = array_reader.next_batch(values_per_page).unwrap();
- assert_eq!(array.len(), values_per_page / 2);
- assert_eq!(
- Some(&def_levels[accu_len..(accu_len + array.len())]),
- array_reader.get_def_levels()
- );
- assert_eq!(
- Some(&rep_levels[accu_len..(accu_len + array.len())]),
- array_reader.get_rep_levels()
- );
- }
-}
diff --git a/parquet/src/arrow/arrow_reader.rs b/parquet/src/arrow/arrow_reader.rs
deleted file mode 100644
index 83fb0a2..0000000
--- a/parquet/src/arrow/arrow_reader.rs
+++ /dev/null
@@ -1,671 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-//! Contains reader which reads parquet data into arrow array.
-
-use crate::arrow::array_reader::{build_array_reader, ArrayReader, StructArrayReader};
-use crate::arrow::schema::parquet_to_arrow_schema;
-use crate::arrow::schema::{
- parquet_to_arrow_schema_by_columns, parquet_to_arrow_schema_by_root_columns,
-};
-use crate::errors::{ParquetError, Result};
-use crate::file::metadata::ParquetMetaData;
-use crate::file::reader::FileReader;
-use arrow::datatypes::{DataType as ArrowType, Schema, SchemaRef};
-use arrow::error::Result as ArrowResult;
-use arrow::record_batch::{RecordBatch, RecordBatchReader};
-use arrow::{array::StructArray, error::ArrowError};
-use std::sync::Arc;
-
-/// Arrow reader api.
-/// With this api, user can get arrow schema from parquet file, and read parquet data
-/// into arrow arrays.
-pub trait ArrowReader {
- type RecordReader: RecordBatchReader;
-
- /// Read parquet schema and convert it into arrow schema.
- fn get_schema(&mut self) -> Result<Schema>;
-
- /// Read parquet schema and convert it into arrow schema.
- /// This schema only includes columns identified by `column_indices`.
- /// To select leaf columns (i.e. `a.b.c` instead of `a`), set `leaf_columns = true`
- fn get_schema_by_columns<T>(
- &mut self,
- column_indices: T,
- leaf_columns: bool,
- ) -> Result<Schema>
- where
- T: IntoIterator<Item = usize>;
-
- /// Returns record batch reader from whole parquet file.
- ///
- /// # Arguments
- ///
- /// `batch_size`: The size of each record batch returned from this reader. Only the
- /// last batch may contain records less than this size, otherwise record batches
- /// returned from this reader should contains exactly `batch_size` elements.
- fn get_record_reader(&mut self, batch_size: usize) -> Result<Self::RecordReader>;
-
- /// Returns record batch reader whose record batch contains columns identified by
- /// `column_indices`.
- ///
- /// # Arguments
- ///
- /// `column_indices`: The columns that should be included in record batches.
- /// `batch_size`: Please refer to `get_record_reader`.
- fn get_record_reader_by_columns<T>(
- &mut self,
- column_indices: T,
- batch_size: usize,
- ) -> Result<Self::RecordReader>
- where
- T: IntoIterator<Item = usize>;
-}
-
-pub struct ParquetFileArrowReader {
- file_reader: Arc<dyn FileReader>,
-}
-
-impl ArrowReader for ParquetFileArrowReader {
- type RecordReader = ParquetRecordBatchReader;
-
- fn get_schema(&mut self) -> Result<Schema> {
- let file_metadata = self.file_reader.metadata().file_metadata();
- parquet_to_arrow_schema(
- file_metadata.schema_descr(),
- file_metadata.key_value_metadata(),
- )
- }
-
- fn get_schema_by_columns<T>(
- &mut self,
- column_indices: T,
- leaf_columns: bool,
- ) -> Result<Schema>
- where
- T: IntoIterator<Item = usize>,
- {
- let file_metadata = self.file_reader.metadata().file_metadata();
- if leaf_columns {
- parquet_to_arrow_schema_by_columns(
- file_metadata.schema_descr(),
- column_indices,
- file_metadata.key_value_metadata(),
- )
- } else {
- parquet_to_arrow_schema_by_root_columns(
- file_metadata.schema_descr(),
- column_indices,
- file_metadata.key_value_metadata(),
- )
- }
- }
-
- fn get_record_reader(
- &mut self,
- batch_size: usize,
- ) -> Result<ParquetRecordBatchReader> {
- let column_indices = 0..self
- .file_reader
- .metadata()
- .file_metadata()
- .schema_descr()
- .num_columns();
-
- self.get_record_reader_by_columns(column_indices, batch_size)
- }
-
- fn get_record_reader_by_columns<T>(
- &mut self,
- column_indices: T,
- batch_size: usize,
- ) -> Result<ParquetRecordBatchReader>
- where
- T: IntoIterator<Item = usize>,
- {
- let array_reader = build_array_reader(
- self.file_reader
- .metadata()
- .file_metadata()
- .schema_descr_ptr(),
- self.get_schema()?,
- column_indices,
- self.file_reader.clone(),
- )?;
-
- ParquetRecordBatchReader::try_new(batch_size, array_reader)
- }
-}
-
-impl ParquetFileArrowReader {
- pub fn new(file_reader: Arc<dyn FileReader>) -> Self {
- Self { file_reader }
- }
-
- // Expose the reader metadata
- pub fn get_metadata(&mut self) -> ParquetMetaData {
- self.file_reader.metadata().clone()
- }
-}
-
-pub struct ParquetRecordBatchReader {
- batch_size: usize,
- array_reader: Box<dyn ArrayReader>,
- schema: SchemaRef,
-}
-
-impl Iterator for ParquetRecordBatchReader {
- type Item = ArrowResult<RecordBatch>;
-
- fn next(&mut self) -> Option<Self::Item> {
- match self.array_reader.next_batch(self.batch_size) {
- Err(error) => Some(Err(error.into())),
- Ok(array) => {
- let struct_array =
- array.as_any().downcast_ref::<StructArray>().ok_or_else(|| {
- ArrowError::ParquetError(
- "Struct array reader should return struct array".to_string(),
- )
- });
- match struct_array {
- Err(err) => Some(Err(err)),
- Ok(e) => {
- match RecordBatch::try_new(self.schema.clone(), e.columns_ref()) {
- Err(err) => Some(Err(err)),
- Ok(record_batch) => {
- if record_batch.num_rows() > 0 {
- Some(Ok(record_batch))
- } else {
- None
- }
- }
- }
- }
- }
- }
- }
- }
-}
-
... 41122 lines suppressed ...