You are viewing a plain text version of this content. The canonical link for it is here.
Posted to github@arrow.apache.org by "mapleFU (via GitHub)" <gi...@apache.org> on 2026/03/24 13:40:13 UTC
Re: [PR] GH-49376: [Python][Parquet] Add ability to write Bloom filters from pyarrow [arrow]
mapleFU commented on code in PR #49377:
URL: https://github.com/apache/arrow/pull/49377#discussion_r2981617188
##########
python/pyarrow/_parquet.pyx:
##########
@@ -2122,6 +2128,48 @@ cdef shared_ptr[WriterProperties] _create_writer_properties(
raise TypeError(
"'column_encoding' should be a dictionary or a string")
+ # bloom filters
+ if bloom_filter_options is not None:
+ if isinstance(bloom_filter_options, dict):
+ # for each entry in bloom_filter_options, {"path": {"ndv": ndv, "fpp", fpp}}
+ # convert (ndv,fpp) to BloomFilterOptions struct and pass to props
+ for column, _bloom_opts in bloom_filter_options.items():
+ # set defaults
+ bloom_opts.ndv = _DEFAULT_BLOOM_FILTER_NDV
+ bloom_opts.fpp = _DEFAULT_BLOOM_FILTER_FPP
+ if isinstance(_bloom_opts, dict):
+ if "ndv" in _bloom_opts:
+ ndv = _bloom_opts["ndv"]
+ if isinstance(ndv, int):
+ if ndv < 0:
+ raise ValueError(
+ f"'ndv' for column '{column}' must be positive, got {ndv}")
+ bloom_opts.ndv = ndv
+ else:
+ raise TypeError(
+ f"'ndv' for column '{column}' must be an int")
+ if "fpp" in _bloom_opts:
+ fpp = _bloom_opts["fpp"]
+ if isinstance(fpp, float):
+ if fpp <= 0.0 or fpp >= 1.0:
+ raise ValueError(
+ f"'fpp' for column '{column}' must be in (0.0, 1,0), got {fpp}")
Review Comment:
1,0 typo here?
##########
python/pyarrow/_parquet.pyx:
##########
@@ -2122,6 +2128,48 @@ cdef shared_ptr[WriterProperties] _create_writer_properties(
raise TypeError(
"'column_encoding' should be a dictionary or a string")
+ # bloom filters
+ if bloom_filter_options is not None:
+ if isinstance(bloom_filter_options, dict):
+ # for each entry in bloom_filter_options, {"path": {"ndv": ndv, "fpp", fpp}}
+ # convert (ndv,fpp) to BloomFilterOptions struct and pass to props
+ for column, _bloom_opts in bloom_filter_options.items():
+ # set defaults
+ bloom_opts.ndv = _DEFAULT_BLOOM_FILTER_NDV
+ bloom_opts.fpp = _DEFAULT_BLOOM_FILTER_FPP
+ if isinstance(_bloom_opts, dict):
+ if "ndv" in _bloom_opts:
+ ndv = _bloom_opts["ndv"]
+ if isinstance(ndv, int):
+ if ndv < 0:
Review Comment:
ndv <= 0?
##########
python/pyarrow/_parquet.pyx:
##########
@@ -2122,6 +2128,48 @@ cdef shared_ptr[WriterProperties] _create_writer_properties(
raise TypeError(
"'column_encoding' should be a dictionary or a string")
+ # bloom filters
+ if bloom_filter_options is not None:
+ if isinstance(bloom_filter_options, dict):
+ # for each entry in bloom_filter_options, {"path": {"ndv": ndv, "fpp", fpp}}
+ # convert (ndv,fpp) to BloomFilterOptions struct and pass to props
+ for column, _bloom_opts in bloom_filter_options.items():
+ # set defaults
+ bloom_opts.ndv = _DEFAULT_BLOOM_FILTER_NDV
+ bloom_opts.fpp = _DEFAULT_BLOOM_FILTER_FPP
+ if isinstance(_bloom_opts, dict):
+ if "ndv" in _bloom_opts:
+ ndv = _bloom_opts["ndv"]
+ if isinstance(ndv, int):
+ if ndv < 0:
+ raise ValueError(
+ f"'ndv' for column '{column}' must be positive, got {ndv}")
+ bloom_opts.ndv = ndv
+ else:
+ raise TypeError(
+ f"'ndv' for column '{column}' must be an int")
+ if "fpp" in _bloom_opts:
+ fpp = _bloom_opts["fpp"]
+ if isinstance(fpp, float):
+ if fpp <= 0.0 or fpp >= 1.0:
+ raise ValueError(
+ f"'fpp' for column '{column}' must be in (0.0, 1,0), got {fpp}")
+ bloom_opts.fpp = fpp
+ else:
+ raise TypeError(
+ f"'fpp' for column '{column}' must be a float")
+ elif isinstance(_bloom_opts, bool):
+ if not _bloom_opts:
Review Comment:
Add comments for `_bloom_opts == True`?
##########
python/pyarrow/_parquet.pyx:
##########
@@ -2122,6 +2128,48 @@ cdef shared_ptr[WriterProperties] _create_writer_properties(
raise TypeError(
"'column_encoding' should be a dictionary or a string")
+ # bloom filters
+ if bloom_filter_options is not None:
+ if isinstance(bloom_filter_options, dict):
+ # for each entry in bloom_filter_options, {"path": {"ndv": ndv, "fpp", fpp}}
+ # convert (ndv,fpp) to BloomFilterOptions struct and pass to props
+ for column, _bloom_opts in bloom_filter_options.items():
+ # set defaults
+ bloom_opts.ndv = _DEFAULT_BLOOM_FILTER_NDV
+ bloom_opts.fpp = _DEFAULT_BLOOM_FILTER_FPP
+ if isinstance(_bloom_opts, dict):
+ if "ndv" in _bloom_opts:
+ ndv = _bloom_opts["ndv"]
+ if isinstance(ndv, int):
+ if ndv < 0:
+ raise ValueError(
+ f"'ndv' for column '{column}' must be positive, got {ndv}")
+ bloom_opts.ndv = ndv
+ else:
+ raise TypeError(
+ f"'ndv' for column '{column}' must be an int")
Review Comment:
should we notice they're in bloom filter options?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscribe@arrow.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org