You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datasketches.apache.org by jm...@apache.org on 2024/01/13 02:21:43 UTC
(datasketches-python) 01/04: Add static/non-statics for most classes, add extra description for serdes
This is an automated email from the ASF dual-hosted git repository.
jmalkin pushed a commit to branch expand_docs
in repository https://gitbox.apache.org/repos/asf/datasketches-python.git
commit 661ed3432c5c1647b0111421c4d0a47b5c6a49fe
Author: Jon Malkin <78...@users.noreply.github.com>
AuthorDate: Fri Jan 12 14:09:21 2024 -0800
Add static/non-statics for most classes, add extra description for serdes
---
datasketches/PySerDe.py | 26 ++++++++++++-------------
docs/source/count_min_sketch.rst | 1 -
docs/source/cpc.rst | 12 ++++++++++--
docs/source/density_sketch.rst | 18 ++++++++++++++---
docs/source/frequent_items.rst | 32 +++++++++++++++++++++++++++++-
docs/source/hyper_log_log.rst | 39 +++++++++++++++++++++++++++++--------
docs/source/index.rst | 39 ++++++++++++++++++++++++-------------
docs/source/kll.rst | 42 ++++++++++++++++++++++++++++++++++++++--
docs/source/quantiles_depr.rst | 41 +++++++++++++++++++++++++++++++++++++++
docs/source/req.rst | 32 +++++++++++++++++++++++++++++-
src/fi_wrapper.cpp | 4 ++--
src/py_serde.cpp | 17 +++++++++++++---
12 files changed, 254 insertions(+), 49 deletions(-)
diff --git a/datasketches/PySerDe.py b/datasketches/PySerDe.py
index 9a24484..6c3a926 100644
--- a/datasketches/PySerDe.py
+++ b/datasketches/PySerDe.py
@@ -32,12 +32,12 @@ import struct
# returns a tuple with the newly reconstructed object and the
# total number of bytes beyond the offset read from the input data.
-# Implements a simple string-encoding scheme where a string is
-# written as <num_bytes> <string_contents>, with no null termination.
-# This format allows pre-allocating each string, at the cost of
-# additional storage. Using this format, the serialized string consumes
-# 4 + len(item) bytes.
class PyStringsSerDe(PyObjectSerDe):
+ '''Implements a simple string-encoding scheme where a string is
+ written as <num_bytes> <string_contents>, with no null termination.
+ This format allows pre-allocating each string, at the cost of
+ additional storage. Using this format, the serialized string consumes
+ 4 + len(item) bytes.'''
def get_size(self, item):
return int(4 + len(item))
@@ -54,9 +54,9 @@ class PyStringsSerDe(PyObjectSerDe):
str = data[offset+4:offset+4+num_chars].decode()
return (str, 4+num_chars)
-# Implements an integer encoding scheme where each integer is written
-# as a 32-bit (4 byte) little-endian value.
class PyIntsSerDe(PyObjectSerDe):
+ '''Implements an integer encoding scheme where each integer is written
+ as a 32-bit (4 byte) little-endian value.'''
def get_size(self, item):
return int(4)
@@ -68,9 +68,9 @@ class PyIntsSerDe(PyObjectSerDe):
return (val, 4)
-# Implements an integer encoding scheme where each integer is written
-# as a 64-bit (8 byte) little-endian value.
class PyLongsSerDe(PyObjectSerDe):
+ '''Implements an integer encoding scheme where each integer is written
+ as a 64-bit (8 byte) little-endian value.'''
def get_size(self, item):
return int(8)
@@ -82,9 +82,9 @@ class PyLongsSerDe(PyObjectSerDe):
return (val, 8)
-# Implements a floating point encoding scheme where each value is written
-# as a 32-bit floating point value.
class PyFloatsSerDe(PyObjectSerDe):
+ '''Implements a floating point encoding scheme where each value is written
+ as a 32-bit floating point value.'''
def get_size(self, item):
return int(4)
@@ -96,9 +96,9 @@ class PyFloatsSerDe(PyObjectSerDe):
return (val, 4)
-# Implements a floating point encoding scheme where each value is written
-# as a 64-bit floating point value.
class PyDoublesSerDe(PyObjectSerDe):
+ '''Implements a floating point encoding scheme where each value is written
+ as a 64-bit floating point value.'''
def get_size(self, item):
return int(8)
diff --git a/docs/source/count_min_sketch.rst b/docs/source/count_min_sketch.rst
index 2f66c8d..b53b02e 100644
--- a/docs/source/count_min_sketch.rst
+++ b/docs/source/count_min_sketch.rst
@@ -17,7 +17,6 @@ heavy hitters.
:members:
:undoc-members:
:exclude-members: deserialize, suggest_num_buckets, suggest_num_hashes
- :member-order: groupwise
.. rubric:: Static Methods:
diff --git a/docs/source/cpc.rst b/docs/source/cpc.rst
index 2959b02..6a28d38 100644
--- a/docs/source/cpc.rst
+++ b/docs/source/cpc.rst
@@ -13,8 +13,7 @@ For additional security this sketch can be configured with a user-specified hash
.. autoclass:: _datasketches.cpc_sketch
:members:
:undoc-members:
- :exclude-members: deserialize,
- :member-order: groupwise
+ :exclude-members: deserialize
.. rubric:: Static Methods:
@@ -22,3 +21,12 @@ For additional security this sketch can be configured with a user-specified hash
.. rubric:: Non-static Methods:
+ .. automethod:: __init__
+
+
+.. autoclass:: _datasketches.cpc_union
+ :members:
+ :undoc-members:
+ :exclude-members: deserialize
+
+ .. automethod:: __init__
diff --git a/docs/source/density_sketch.rst b/docs/source/density_sketch.rst
index 8326ebf..529767d 100644
--- a/docs/source/density_sketch.rst
+++ b/docs/source/density_sketch.rst
@@ -9,9 +9,21 @@ https://proceedings.mlr.press/v99/karnin19a/karnin19a.pdf
Inspired by the following implementation: https://github.com/edoliberty/streaming-quantiles/blob/f688c8161a25582457b0a09deb4630a81406293b/gde.py
+Requires the use of a `KernelFunction` to compute the distance between two vectors.
+
+.. autoclass:: datasketches.KernelFunction
+
+ .. automethod:: __call__
+
.. autoclass:: datasketches.density_sketch
:members:
:undoc-members:
-
-.. autoclass:: datasketches.GaussianKernel
- :members:
\ No newline at end of file
+ :exclude-members: deserialize
+
+ .. rubric:: Static Methods:
+
+ .. automethod:: deserialize
+
+ .. rubric:: Non-static Methods:
+
+ .. automethod:: __init__
diff --git a/docs/source/frequent_items.rst b/docs/source/frequent_items.rst
index bb580e5..6fef7ae 100644
--- a/docs/source/frequent_items.rst
+++ b/docs/source/frequent_items.rst
@@ -65,7 +65,18 @@ However, we have ensured that this probability is extremely small.
For example, if the stream causes one table purge (rebuild), our proof of the worst-case bound applies with a probability of at least `1 - 1E-14`.
If the stream causes `1E9` purges, our proof applies with a probability of at least `1 - 1E-5`.
-Parameter: <T> The type of item to be tracked by this sketch
+There are two flavors of Frequent Items Sketches, one with generic items (objects) and another specific to strings.
+The string version is a legacy name from before the library supported generic objects and is retained
+only for backwards compatibility.
+
+
+.. autoclass:: _datasketches.frequent_items_error_type
+
+ .. autoattribute:: NO_FALSE_POSITIVES
+ :annotation: : Returns only true positives but may miss some heavy hitters.
+
+ .. autoattribute:: NO_FALSE_NEGATIVES
+ :annotation: : Does not miss any heavy hitters but may return false positives.
.. autoclass:: _datasketches.frequent_items_sketch
@@ -81,3 +92,22 @@ Parameter: <T> The type of item to be tracked by this sketch
.. automethod:: get_apriori_error
.. rubric:: Non-static Methods:
+
+ .. automethod:: __init__
+
+
+.. autoclass:: _datasketches.frequent_strings_sketch
+ :members:
+ :undoc-members:
+ :exclude-members: deserialize, get_epsilon_for_lg_size, get_apriori_error
+ :member-order: groupwise
+
+ .. rubric:: Static Methods:
+
+ .. automethod:: deserialize
+ .. automethod:: get_epsilon_for_lg_size
+ .. automethod:: get_apriori_error
+
+ .. rubric:: Non-static Methods:
+
+ .. automethod:: __init__
diff --git a/docs/source/hyper_log_log.rst b/docs/source/hyper_log_log.rst
index ec79d93..3e4f985 100644
--- a/docs/source/hyper_log_log.rst
+++ b/docs/source/hyper_log_log.rst
@@ -7,22 +7,29 @@ If the ONLY use case for sketching is counting uniques and merging, the HLL sket
This implementation offers three different types of HLL sketch, each with different trade-offs with accuracy, space and performance.
These types are specified with the target_hll_type parameter.
-In terms of accuracy, all three types, for the same lg_config_k, have the same error distribution as a function of n, the number of unique values fed to the sketch.
-The configuration parameter `lg_config_k` is the log-base-2 of `K`, where `K` is the number of buckets or slots for the sketch.
+In terms of accuracy, all three types, for the same lg_config_k, have the same error distribution as a function of `n`, the number of unique values fed to the sketch.
+The configuration parameter `lg_config_k` is the log-base-2 of `k`, where `k`` is the number of buckets or slots for the sketch.
-During warmup, when the sketch has only received a small number of unique items (up to about 10% of `K`), this implementation leverages a new class of estimator algorithms with significantly better accuracy.
+During warmup, when the sketch has only received a small number of unique items (up to about 10% of `k`), this implementation leverages a new class of estimator algorithms with significantly better accuracy.
+
+
+.. autoclass:: _datasketches.tgt_hll_type
+
+ .. autoattribute:: HLL_4
+ :annotation: : 4 bits per entry
+
+ .. autoattribute:: HLL_6
+ :annotation: : 6 bits per entry
+
+ .. autoattribute:: HLL_8
+ :annotation: : 8 bits per entry
-This sketch also offers the capability of operating off-heap.
-Given a WritableMemory object created by the user, the sketch will perform all of its updates and internal phase transitions in that object, which can actually reside either on-heap or off-heap based on how it is configured.
-In large systems that must update and merge many millions of sketches, having the sketch operate off-heap avoids the serialization and deserialization costs of moving sketches to and from off-heap memory-mapped files, for example, and eliminates big garbage collection delays.
.. autoclass:: _datasketches.hll_sketch
:members:
:undoc-members:
:exclude-members: deserialize, get_max_updatable_serialization_bytes, get_rel_err
- :member-order: groupwise
-
.. rubric:: Static Methods:
.. automethod:: deserialize
@@ -30,3 +37,19 @@ In large systems that must update and merge many millions of sketches, having th
.. automethod:: get_rel_err
.. rubric:: Non-static Methods:
+
+ .. automethod:: __init__
+
+.. autoclass:: _datasketches.hll_union
+ :members:
+ :undoc-members:
+ :exclude-members: get_rel_err
+
+ .. rubric:: Static Methods:
+
+ .. automethod:: get_rel_err
+
+ .. rubric:: Non-static Methods:
+
+ .. automethod:: __init__
+
\ No newline at end of file
diff --git a/docs/source/index.rst b/docs/source/index.rst
index b6998c2..a44e873 100644
--- a/docs/source/index.rst
+++ b/docs/source/index.rst
@@ -55,7 +55,32 @@ Quantile Estimation
kll
req
quantiles_depr
-
+
+
+Random Sampling
+###############
+
+.. toctree::
+ :maxdepth: 1
+
+ varopt
+ ebpps
+
+Helper Classes
+##############
+
+.. toctree::
+ :maxdepth: 1
+
+ serde
+
+These classes are required for certain sketches or specific functionality withing sketches.
+Some of them are abstract base classes, but in those cases there is at least one reference example of a
+concrete class.
+
+
+
+
.. note::
This project is under active development.
@@ -67,15 +92,3 @@ Indices and tables
* :ref:`genindex`
* :ref:`modindex`
* :ref:`search`
-
-
-.. .. automodule:: datasketches
-.. :members:
-
-.. .. automodule:: _datasketches
-.. :members:
-
-..
-..
-
-.. distinct_count
diff --git a/docs/source/kll.rst b/docs/source/kll.rst
index 1be3dfa..58b4718 100644
--- a/docs/source/kll.rst
+++ b/docs/source/kll.rst
@@ -104,21 +104,59 @@ Additionally, the interval may be quite large for certain distributions.
- Then `q_lo ≤ q ≤ q_hi`, with 99% confidence.
-
-
.. autoclass:: _datasketches.kll_ints_sketch
:members:
:undoc-members:
+ :exclude-members: deserialize, get_normalized_rank_error
+
+ .. rubric:: Static Methods:
+
+ .. automethod:: deserialize
+ .. automethod:: get_normalized_rank_error
+
+ .. rubric:: Non-static Methods:
+
+ .. automethod:: __init__
.. autoclass:: _datasketches.kll_floats_sketch
:members:
:undoc-members:
+ :exclude-members: deserialize, get_normalized_rank_error
+
+ .. rubric:: Static Methods:
+
+ .. automethod:: deserialize
+ .. automethod:: get_normalized_rank_error
+
+ .. rubric:: Non-static Methods:
+
+ .. automethod:: __init__
.. autoclass:: _datasketches.kll_doubles_sketch
:members:
:undoc-members:
+ :exclude-members: deserialize, get_normalized_rank_error
+
+ .. rubric:: Static Methods:
+
+ .. automethod:: deserialize
+ .. automethod:: get_normalized_rank_error
+
+ .. rubric:: Non-static Methods:
+
+ .. automethod:: __init__
.. autoclass:: _datasketches.kll_items_sketch
:members:
:undoc-members:
+ :exclude-members: deserialize, get_normalized_rank_error
+
+ .. rubric:: Static Methods:
+
+ .. automethod:: deserialize
+ .. automethod:: get_normalized_rank_error
+
+ .. rubric:: Non-static Methods:
+
+ .. automethod:: __init__
diff --git a/docs/source/quantiles_depr.rst b/docs/source/quantiles_depr.rst
index a2d12ea..84c0708 100644
--- a/docs/source/quantiles_depr.rst
+++ b/docs/source/quantiles_depr.rst
@@ -1,6 +1,7 @@
Quantiles Sketch (Deprecated)
-----------------------------
This is a deprecated quantiles sketch that is included for cross-language compatibility.
+Most new projects will favor the KLL sketch over this one.
This is a stochastic streaming sketch that enables near-real time analysis of the
approximate distribution from a very large stream in a single pass.
@@ -41,15 +42,55 @@ a confidence of about 99%.
.. autoclass:: _datasketches.quantiles_ints_sketch
:members:
:undoc-members:
+ :exclude-members: deserialize, get_normalized_rank_error
+
+ .. rubric:: Static Methods:
+
+ .. automethod:: deserialize
+ .. automethod:: get_normalized_rank_error
+
+ .. rubric:: Non-static Methods:
+
+ .. automethod:: __init__
.. autoclass:: _datasketches.quantiles_floats_sketch
:members:
:undoc-members:
+ :exclude-members: deserialize, get_normalized_rank_error
+
+ .. rubric:: Static Methods:
+
+ .. automethod:: deserialize
+ .. automethod:: get_normalized_rank_error
+
+ .. rubric:: Non-static Methods:
+
+ .. automethod:: __init__
.. autoclass:: _datasketches.quantiles_doubles_sketch
:members:
:undoc-members:
+ :exclude-members: deserialize, get_normalized_rank_error
+
+ .. rubric:: Static Methods:
+
+ .. automethod:: deserialize
+ .. automethod:: get_normalized_rank_error
+
+ .. rubric:: Non-static Methods:
+
+ .. automethod:: __init__
.. autoclass:: _datasketches.quantiles_items_sketch
:members:
:undoc-members:
+ :exclude-members: deserialize, get_normalized_rank_error
+
+ .. rubric:: Static Methods:
+
+ .. automethod:: deserialize
+ .. automethod:: get_normalized_rank_error
+
+ .. rubric:: Non-static Methods:
+
+ .. automethod:: __init__
diff --git a/docs/source/req.rst b/docs/source/req.rst
index 1df4d08..48978c5 100644
--- a/docs/source/req.rst
+++ b/docs/source/req.rst
@@ -30,11 +30,41 @@ This is not only useful for debugging, but is a powerful tool to help users unde
.. autoclass:: _datasketches.req_ints_sketch
:members:
:undoc-members:
+ :exclude-members: deserialize, get_RSE
+
+ .. rubric:: Static Methods:
+
+ .. automethod:: deserialize
+ .. automethod:: get_RSE
+
+ .. rubric:: Non-static Methods:
+
+ .. automethod:: __init__
.. autoclass:: _datasketches.req_floats_sketch
:members:
:undoc-members:
+ :exclude-members: deserialize, get_RSE
+
+ .. rubric:: Static Methods:
+
+ .. automethod:: deserialize
+ .. automethod:: get_RSE
+
+ .. rubric:: Non-static Methods:
+
+ .. automethod:: __init__
.. autoclass:: _datasketches.req_items_sketch
:members:
- :undoc-members:
\ No newline at end of file
+ :undoc-members:
+ :exclude-members: deserialize, get_RSE
+
+ .. rubric:: Static Methods:
+
+ .. automethod:: deserialize
+ .. automethod:: get_RSE
+
+ .. rubric:: Non-static Methods:
+
+ .. automethod:: __init__
diff --git a/src/fi_wrapper.cpp b/src/fi_wrapper.cpp
index eab848c..5cfb82c 100644
--- a/src/fi_wrapper.cpp
+++ b/src/fi_wrapper.cpp
@@ -186,8 +186,8 @@ void init_fi(nb::module_ &m) {
using namespace datasketches;
nb::enum_<frequent_items_error_type>(m, "frequent_items_error_type")
- .value("NO_FALSE_POSITIVES", NO_FALSE_POSITIVES)
- .value("NO_FALSE_NEGATIVES", NO_FALSE_NEGATIVES)
+ .value("NO_FALSE_POSITIVES", NO_FALSE_POSITIVES, "Returns only true positives, but may miss some heavy hitters.")
+ .value("NO_FALSE_NEGATIVES", NO_FALSE_NEGATIVES, "Does not miss any heavy hitters, but may return false positives.")
.export_values();
bind_fi_sketch<std::string, uint64_t, std::hash<std::string>, std::equal_to<std::string>>(m, "frequent_strings_sketch");
diff --git a/src/py_serde.cpp b/src/py_serde.cpp
index 6aee282..3aa0f87 100644
--- a/src/py_serde.cpp
+++ b/src/py_serde.cpp
@@ -32,12 +32,23 @@ void init_serde(nb::module_& m) {
"An abstract base class for serde objects. All custom serdes must extend this class.")
.def(nb::init<>())
.def("get_size", &py_object_serde::get_size, nb::arg("item"),
- "Returns the size in bytes of an item")
+ "Returns the size in bytes of an item\n\n"
+ ":param item: The specified object\n:type item: object\n"
+ ":return: The size of the item, in bytes\n:rtype: int"
+ )
.def("to_bytes", &py_object_serde::to_bytes, nb::arg("item"),
- "Retuns a bytes object with a serialized version of an item")
+ "Retuns a bytes object with a serialized version of an item\n\n"
+ ":param item: The specified object\n:type item: object\n"
+ ":return: A bytes object with the serialized object\n:rtype: bytes"
+ )
.def("from_bytes", &py_object_serde::from_bytes, nb::arg("data"), nb::arg("offset"),
"Reads a bytes object starting from the given offest and returns a tuple of the reconstructed "
- "object and the number of additional bytes read")
+ "object and the number of additional bytes read\n\n"
+ ":param data: A bytes object to deserialize\n:type data: bytes\n"
+ ":param offset: The offset, in bytes, at which to start reading\n:type offset: int\n"
+ ":return: A tuple with the reconstructed object and the number of bytes read\n"
+ ":rtype: tuple(object, int)"
+ )
;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org
For additional commands, e-mail: commits-help@datasketches.apache.org