You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datasketches.apache.org by al...@apache.org on 2022/10/06 20:58:29 UTC
[datasketches-cpp] branch universal_sorted_view updated: removed get_quantiles()
This is an automated email from the ASF dual-hosted git repository.
alsay pushed a commit to branch universal_sorted_view
in repository https://gitbox.apache.org/repos/asf/datasketches-cpp.git
The following commit(s) were added to refs/heads/universal_sorted_view by this push:
new 9b2c4fd removed get_quantiles()
9b2c4fd is described below
commit 9b2c4fdfe8216484db1d6cd591b5fe1957732a7e
Author: AlexanderSaydakov <Al...@users.noreply.github.com>
AuthorDate: Thu Oct 6 13:58:22 2022 -0700
removed get_quantiles()
---
common/include/quantile_sketch_sorted_view.hpp | 1 +
kll/include/kll_sketch.hpp | 3 +-
python/src/kll_wrapper.cpp | 18 ------------
python/src/quantiles_wrapper.cpp | 18 ------------
python/src/req_wrapper.cpp | 18 ------------
python/src/vector_of_kll.cpp | 3 +-
python/tests/kll_test.py | 11 -------
python/tests/quantiles_test.py | 11 -------
python/tests/req_test.py | 12 --------
quantiles/include/quantiles_sketch.hpp | 3 +-
quantiles/include/quantiles_sketch_impl.hpp | 36 -----------------------
quantiles/test/quantiles_sketch_test.cpp | 40 --------------------------
req/include/req_sketch.hpp | 3 +-
13 files changed, 8 insertions(+), 169 deletions(-)
diff --git a/common/include/quantile_sketch_sorted_view.hpp b/common/include/quantile_sketch_sorted_view.hpp
index 996a841..198d312 100755
--- a/common/include/quantile_sketch_sorted_view.hpp
+++ b/common/include/quantile_sketch_sorted_view.hpp
@@ -21,6 +21,7 @@
#define QUANTILE_SKETCH_SORTED_VIEW_HPP_
#include <functional>
+#include <cmath>
namespace datasketches {
diff --git a/kll/include/kll_sketch.hpp b/kll/include/kll_sketch.hpp
index 715717c..8895c09 100644
--- a/kll/include/kll_sketch.hpp
+++ b/kll/include/kll_sketch.hpp
@@ -527,7 +527,8 @@ class kll_sketch {
// implementations for floating point types
template<typename TT = T, typename std::enable_if<std::is_floating_point<TT>::value, int>::type = 0>
static const TT& get_invalid_item() {
- return std::numeric_limits<TT>::quiet_NaN();
+ static TT item = std::numeric_limits<TT>::quiet_NaN();
+ return item;
}
template<typename TT = T, typename std::enable_if<std::is_floating_point<TT>::value, int>::type = 0>
diff --git a/python/src/kll_wrapper.cpp b/python/src/kll_wrapper.cpp
index b99340b..ee2cd93 100644
--- a/python/src/kll_wrapper.cpp
+++ b/python/src/kll_wrapper.cpp
@@ -51,20 +51,6 @@ double kll_sketch_generic_normalized_rank_error(uint16_t k, bool pmf) {
return kll_sketch<T>::get_normalized_rank_error(k, pmf);
}
-template<typename T>
-py::list kll_sketch_get_quantiles(const kll_sketch<T>& sk,
- std::vector<double>& ranks,
- bool inclusive) {
- size_t nQuantiles = ranks.size();
- auto result = sk.get_quantiles(ranks.data(), nQuantiles, inclusive);
- // returning as std::vector<> would copy values to a list anyway
- py::list list(nQuantiles);
- for (size_t i = 0; i < nQuantiles; ++i) {
- list[i] = result[i];
- }
- return list;
-}
-
template<typename T>
py::list kll_sketch_get_pmf(const kll_sketch<T>& sk,
std::vector<T>& split_points,
@@ -146,10 +132,6 @@ void bind_kll_sketch(py::module &m, const char* name) {
"version of the input stream so far.\n"
"For kll_floats_sketch: if the sketch is empty this returns nan. "
"For kll_ints_sketch: if the sketch is empty this throws a RuntimeError.")
- .def("get_quantiles", &dspy::kll_sketch_get_quantiles<T>, py::arg("ranks"), py::arg("inclusive")=false,
- "This returns an array that could have been generated by using get_quantile() for each "
- "normalized rank separately.\n"
- "If the sketch is empty this returns an empty vector.")
.def("get_rank", &kll_sketch<T>::get_rank, py::arg("value"), py::arg("inclusive")=false,
"Returns an approximation to the normalized rank of the given value from 0 to 1, inclusive.\n"
"The resulting approximation has a probabilistic guarantee that can be obtained from the "
diff --git a/python/src/quantiles_wrapper.cpp b/python/src/quantiles_wrapper.cpp
index e493cc6..b9bfa2a 100644
--- a/python/src/quantiles_wrapper.cpp
+++ b/python/src/quantiles_wrapper.cpp
@@ -49,20 +49,6 @@ double quantiles_sketch_generic_normalized_rank_error(uint16_t k, bool pmf) {
return quantiles_sketch<T>::get_normalized_rank_error(k, pmf);
}
-template<typename T>
-py::list quantiles_sketch_get_quantiles(const quantiles_sketch<T>& sk,
- std::vector<double>& ranks,
- bool inclusive) {
- size_t n_quantiles = ranks.size();
- auto result = sk.get_quantiles(ranks.data(), static_cast<uint32_t>(n_quantiles), inclusive);
- // returning as std::vector<> would copy values to a list anyway
- py::list list(n_quantiles);
- for (size_t i = 0; i < n_quantiles; ++i) {
- list[i] = result[i];
- }
- return list;
-}
-
template<typename T>
py::list quantiles_sketch_get_pmf(const quantiles_sketch<T>& sk,
std::vector<T>& split_points,
@@ -144,10 +130,6 @@ void bind_quantiles_sketch(py::module &m, const char* name) {
"version of the input stream so far.\n"
"For quantiles_floats_sketch: if the sketch is empty this returns nan. "
"For quantiles_ints_sketch: if the sketch is empty this throws a RuntimeError.")
- .def("get_quantiles", &dspy::quantiles_sketch_get_quantiles<T>, py::arg("ranks"), py::arg("inclusive")=false,
- "This returns an array that could have been generated by using get_quantile() for each "
- "normalized rank separately.\n"
- "If the sketch is empty this returns an empty vector.")
.def("get_rank", &quantiles_sketch<T>::get_rank, py::arg("value"), py::arg("inclusive")=false,
"Returns an approximation to the normalized rank of the given value from 0 to 1, inclusive.\n"
"The resulting approximation has a probabilistic guarantee that can be obtained from the "
diff --git a/python/src/req_wrapper.cpp b/python/src/req_wrapper.cpp
index 4d1efad..7267ef0 100644
--- a/python/src/req_wrapper.cpp
+++ b/python/src/req_wrapper.cpp
@@ -51,20 +51,6 @@ double req_sketch_generic_normalized_rank_error(uint16_t k, bool pmf) {
return req_sketch<T>::get_normalized_rank_error(k, pmf);
}
-template<typename T>
-py::list req_sketch_get_quantiles(const req_sketch<T>& sk,
- std::vector<double>& ranks,
- bool inclusive) {
- size_t n_quantiles = ranks.size();
- auto result = sk.get_quantiles(ranks.data(), n_quantiles, inclusive);
- // returning as std::vector<> would copy values to a list anyway
- py::list list(n_quantiles);
- for (size_t i = 0; i < n_quantiles; ++i) {
- list[i] = result[i];
- }
- return list;
-}
-
template<typename T>
py::list req_sketch_get_pmf(const req_sketch<T>& sk,
std::vector<T>& split_points,
@@ -148,10 +134,6 @@ void bind_req_sketch(py::module &m, const char* name) {
"version of the input stream so far.\n"
"For req_floats_sketch: if the sketch is empty this returns nan. "
"For req_ints_sketch: if the sketch is empty this throws a RuntimeError.")
- .def("get_quantiles", &dspy::req_sketch_get_quantiles<T>, py::arg("ranks"), py::arg("inclusive")=false,
- "This returns an array that could have been generated by using get_quantile() for each "
- "normalized rank separately.\n"
- "If the sketch is empty this returns an empty vector.")
.def("get_rank", &req_sketch<T>::get_rank, py::arg("value"), py::arg("inclusive")=false,
"Returns an approximation to the normalized rank of the given value from 0 to 1, inclusive.\n"
"The resulting approximation has a probabilistic guarantee that can be obtained from the "
diff --git a/python/src/vector_of_kll.cpp b/python/src/vector_of_kll.cpp
index 46b1a5e..db30af6 100644
--- a/python/src/vector_of_kll.cpp
+++ b/python/src/vector_of_kll.cpp
@@ -326,9 +326,8 @@ py::array vector_of_kll_sketches<T, C>::get_quantiles(const py::array_t<double>&
std::vector<std::vector<T>> quants(num_sketches, std::vector<T>(num_quantiles));
for (uint32_t i = 0; i < num_sketches; ++i) {
- auto quant = sketches_[inds[i]].get_quantiles(ranks.data(), num_quantiles);
for (size_t j = 0; j < num_quantiles; ++j) {
- quants[i][j] = quant[j];
+ quants[i][j] = sketches_[inds[i]].get_quantile(ranks.data()[j]);
}
}
diff --git a/python/tests/kll_test.py b/python/tests/kll_test.py
index 0f7168b..066f870 100644
--- a/python/tests/kll_test.py
+++ b/python/tests/kll_test.py
@@ -42,14 +42,6 @@ class KllTest(unittest.TestCase):
self.assertGreaterEqual(kll.get_max_value(), kll.get_quantile(0.99))
self.assertGreaterEqual(1.0, kll.get_rank(kll.get_max_value()))
- # we can also extract a list of values at a time,
- # here the values should give us something close to [-2, -1, 0, 1, 2].
- # then get the CDF, which will return something close to
- # the original values used in get_quantiles()
- # finally, can check the normalized rank error bound
- pts = kll.get_quantiles([0.0228, 0.1587, 0.5, 0.8413, 0.9772])
- cdf = kll.get_cdf(pts) # include 1.0 at end to account for all probability mass
- self.assertEqual(len(cdf), len(pts)+1)
err = kll.normalized_rank_error(False)
self.assertEqual(err, kll_floats_sketch.get_normalized_rank_error(k, False))
@@ -102,9 +94,6 @@ class KllTest(unittest.TestCase):
self.assertEqual(len(cdf), 2)
self.assertEqual(kll.get_quantile(0.5), round(n/2))
- quants = kll.get_quantiles([0.25, 0.5, 0.75])
- self.assertIsNotNone(quants)
- self.assertEqual(len(quants), 3)
self.assertEqual(kll.get_rank(round(n/2)), 0.5)
diff --git a/python/tests/quantiles_test.py b/python/tests/quantiles_test.py
index 7e4a40f..267308a 100644
--- a/python/tests/quantiles_test.py
+++ b/python/tests/quantiles_test.py
@@ -42,14 +42,6 @@ class QuantilesTest(unittest.TestCase):
self.assertGreaterEqual(quantiles.get_max_value(), quantiles.get_quantile(0.99))
self.assertGreaterEqual(1.0, quantiles.get_rank(quantiles.get_max_value()))
- # we can also extract a list of values at a time,
- # here the values should give us something close to [-2, -1, 0, 1, 2].
- # then get the CDF, which will return something close to
- # the original values used in get_quantiles()
- # finally, can check the normalized rank error bound
- pts = quantiles.get_quantiles([0.0228, 0.1587, 0.5, 0.8413, 0.9772])
- cdf = quantiles.get_cdf(pts) # include 1.0 at end to account for all probability mass
- self.assertEqual(len(cdf), len(pts)+1)
err = quantiles.normalized_rank_error(False)
self.assertEqual(err, quantiles_floats_sketch.get_normalized_rank_error(k, False))
@@ -103,9 +95,6 @@ class QuantilesTest(unittest.TestCase):
self.assertEqual(len(cdf), 2)
self.assertEqual(quantiles.get_quantile(0.5), round(n/2))
- quants = quantiles.get_quantiles([0.25, 0.5, 0.75])
- self.assertIsNotNone(quants)
- self.assertEqual(len(quants), 3)
self.assertEqual(quantiles.get_rank(round(n/2)), 0.5)
diff --git a/python/tests/req_test.py b/python/tests/req_test.py
index 9bf877d..ac913eb 100644
--- a/python/tests/req_test.py
+++ b/python/tests/req_test.py
@@ -42,15 +42,6 @@ class reqTest(unittest.TestCase):
self.assertGreaterEqual(req.get_max_value(), req.get_quantile(0.99))
self.assertGreaterEqual(1.0, req.get_rank(req.get_max_value()))
- # we can also extract a list of values at a time,
- # here the values should give us something close to [-2, -1, 0, 1, 2].
- # then get the CDF, which will return something close to
- # the original values used in get_quantiles()
- # finally, can check the normalized rank error bound
- pts = req.get_quantiles([0.0228, 0.1587, 0.5, 0.8413, 0.9772])
- cdf = req.get_cdf(pts) # include 1.0 at end to account for all probability mass
- self.assertEqual(len(cdf), len(pts)+1)
-
# For relative error quantiles, the error depends on the actual rank
# so we need to use that to detemrine the bounds
est = req.get_rank(0.999, True)
@@ -102,9 +93,6 @@ class reqTest(unittest.TestCase):
self.assertEqual(len(cdf), 2)
self.assertEqual(req.get_quantile(0.5), round(n/2))
- quants = req.get_quantiles([0.25, 0.5, 0.75])
- self.assertIsNotNone(quants)
- self.assertEqual(len(quants), 3)
self.assertEqual(req.get_rank(round(n/2)), 0.5)
diff --git a/quantiles/include/quantiles_sketch.hpp b/quantiles/include/quantiles_sketch.hpp
index d6efec0..8e0ac43 100644
--- a/quantiles/include/quantiles_sketch.hpp
+++ b/quantiles/include/quantiles_sketch.hpp
@@ -529,7 +529,8 @@ private:
// implementations for floating point types
template<typename TT = T, typename std::enable_if<std::is_floating_point<TT>::value, int>::type = 0>
static const TT& get_invalid_item() {
- return std::numeric_limits<TT>::quiet_NaN();
+ static TT item = std::numeric_limits<TT>::quiet_NaN();
+ return item;
}
template<typename TT = T, typename std::enable_if<std::is_floating_point<TT>::value, int>::type = 0>
diff --git a/quantiles/include/quantiles_sketch_impl.hpp b/quantiles/include/quantiles_sketch_impl.hpp
index 713327b..0d66e68 100644
--- a/quantiles/include/quantiles_sketch_impl.hpp
+++ b/quantiles/include/quantiles_sketch_impl.hpp
@@ -759,42 +759,6 @@ auto quantiles_sketch<T, C, A>::get_quantile(double rank, bool inclusive) const
return sorted_view_->get_quantile(rank, inclusive);
}
-template<typename T, typename C, typename A>
-std::vector<T, A> quantiles_sketch<T, C, A>::get_quantiles(const double* ranks, uint32_t size, bool inclusive) const {
- std::vector<T, A> quantiles(allocator_);
- if (is_empty()) return quantiles;
- quantiles.reserve(size);
-
- // possible side-effect: sorting base buffer
- setup_sorted_view();
-
- for (uint32_t i = 0; i < size; ++i) {
- const double rank = ranks[i];
- if ((rank < 0.0) || (rank > 1.0)) {
- throw std::invalid_argument("Normalized rank cannot be less than 0 or greater than 1");
- }
- quantiles.push_back(sorted_view_->get_quantile(rank, inclusive));
- }
- return quantiles;
-}
-
-template<typename T, typename C, typename A>
-std::vector<T, A> quantiles_sketch<T, C, A>::get_quantiles(uint32_t num, bool inclusive) const {
- if (is_empty()) return std::vector<T, A>(allocator_);
- if (num == 0) {
- throw std::invalid_argument("num must be > 0");
- }
- vector_double ranks(num, 0, allocator_);
- ranks[0] = 0.0;
- for (size_t i = 1; i < num; i++) {
- ranks[i] = static_cast<double>(i) / (num - 1);
- }
- if (num > 1) {
- ranks[num - 1] = 1.0;
- }
- return get_quantiles(ranks.data(), num, inclusive);
-}
-
template<typename T, typename C, typename A>
double quantiles_sketch<T, C, A>::get_rank(const T& item, bool inclusive) const {
if (is_empty()) return std::numeric_limits<double>::quiet_NaN();
diff --git a/quantiles/test/quantiles_sketch_test.cpp b/quantiles/test/quantiles_sketch_test.cpp
index 75c2e89..0447ffc 100644
--- a/quantiles/test/quantiles_sketch_test.cpp
+++ b/quantiles/test/quantiles_sketch_test.cpp
@@ -64,8 +64,6 @@ TEST_CASE("quantiles sketch", "[quantiles_sketch]") {
REQUIRE(std::isnan(sketch.get_min_item()));
REQUIRE(std::isnan(sketch.get_max_item()));
REQUIRE(std::isnan(sketch.get_quantile(0.5)));
- const double fractions[3] {0, 0.5, 1};
- REQUIRE(sketch.get_quantiles(fractions, 3).empty());
const float split_points[1] {0};
REQUIRE(sketch.get_PMF(split_points, 1).empty());
REQUIRE(sketch.get_CDF(split_points, 1).empty());
@@ -96,12 +94,6 @@ TEST_CASE("quantiles sketch", "[quantiles_sketch]") {
REQUIRE(sketch.get_min_item() == 1.0);
REQUIRE(sketch.get_max_item() == 1.0);
REQUIRE(sketch.get_quantile(0.5) == 1.0);
- const double fractions[3] {0, 0.5, 1};
- auto quantiles = sketch.get_quantiles(fractions, 3);
- REQUIRE(quantiles.size() == 3);
- REQUIRE(quantiles[0] == 1.0);
- REQUIRE(quantiles[1] == 1.0);
- REQUIRE(quantiles[2] == 1.0);
int count = 0;
for (auto it: sketch) {
@@ -153,20 +145,6 @@ TEST_CASE("quantiles sketch", "[quantiles_sketch]") {
}
REQUIRE(count == n);
- const double ranks[3] {0, 0.5, 1};
- auto quantiles = sketch.get_quantiles(ranks, 3);
- REQUIRE(quantiles.size() == 3);
- REQUIRE(quantiles[0] == 0.0);
- REQUIRE(quantiles[1] == static_cast<float>(n / 2));
- REQUIRE(quantiles[2] == n - 1 );
-
- // the alternative method must produce the same result
- auto quantiles2 = sketch.get_quantiles(3);
- REQUIRE(quantiles2.size() == 3);
- REQUIRE(quantiles[0] == quantiles2[0]);
- REQUIRE(quantiles[1] == quantiles2[1]);
- REQUIRE(quantiles[2] == quantiles2[2]);
-
for (uint32_t i = 0; i < n; i++) {
const double trueRank = static_cast<double>(i + 1) / n;
REQUIRE(sketch.get_rank(static_cast<float>(i)) == trueRank);
@@ -220,24 +198,6 @@ TEST_CASE("quantiles sketch", "[quantiles_sketch]") {
REQUIRE(sketchRank == Approx(trueRank).margin(RANK_EPS_FOR_K_128));
}
- // test quantiles at every 0.1 percentage point
- double ranks[1001];
- double reverse_ranks[1001]; // check that ordering does not matter
- for (int i = 0; i < 1001; i++) {
- ranks[i] = (double) i / 1000;
- reverse_ranks[1000 - i] = ranks[i];
- }
- auto quantiles = sketch.get_quantiles(ranks, 1001);
- auto reverse_quantiles = sketch.get_quantiles(reverse_ranks, 1001);
- float previous_quantile = 0;
- for (int i = 0; i < 1001; i++) {
- const float quantile = sketch.get_quantile(ranks[i]);
- REQUIRE(quantiles[i] == quantile);
- REQUIRE(reverse_quantiles[1000 - i] == quantile);
- REQUIRE(previous_quantile <= quantile);
- previous_quantile = quantile;
- }
-
//std::cout << sketch.to_string();
uint32_t count = 0;
diff --git a/req/include/req_sketch.hpp b/req/include/req_sketch.hpp
index cab57ac..8cc5fe7 100755
--- a/req/include/req_sketch.hpp
+++ b/req/include/req_sketch.hpp
@@ -357,7 +357,8 @@ private:
// implementations for floating point types
template<typename TT = T, typename std::enable_if<std::is_floating_point<TT>::value, int>::type = 0>
static const TT& get_invalid_item() {
- return std::numeric_limits<TT>::quiet_NaN();
+ static TT item = std::numeric_limits<TT>::quiet_NaN();
+ return item;
}
template<typename TT = T, typename std::enable_if<std::is_floating_point<TT>::value, int>::type = 0>
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org
For additional commands, e-mail: commits-help@datasketches.apache.org