You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datasketches.apache.org by le...@apache.org on 2026/03/27 01:37:43 UTC
(datasketches-java) 01/01: Fixes the getRSE() function and updates the SketchesCheckstyle.xml.
This is an automated email from the ASF dual-hosted git repository.
leerho pushed a commit to branch fix_getRSE
in repository https://gitbox.apache.org/repos/asf/datasketches-java.git
commit 011f775e3f4fd82f863a5d14d63d9029b8b59cfe
Author: Lee Rhodes <le...@gmail.com>
AuthorDate: Thu Mar 26 18:37:25 2026 -0700
Fixes the getRSE() function and updates the SketchesCheckstyle.xml.
---
src/main/java/org/apache/datasketches/req/BaseReqSketch.java | 7 +++----
tools/SketchesCheckstyle.xml | 3 +++
2 files changed, 6 insertions(+), 4 deletions(-)
diff --git a/src/main/java/org/apache/datasketches/req/BaseReqSketch.java b/src/main/java/org/apache/datasketches/req/BaseReqSketch.java
index f9411d5f4..fd44a12df 100644
--- a/src/main/java/org/apache/datasketches/req/BaseReqSketch.java
+++ b/src/main/java/org/apache/datasketches/req/BaseReqSketch.java
@@ -71,7 +71,7 @@ abstract class BaseReqSketch implements QuantilesFloatsAPI {
* @return an a priori estimate of relative standard error (RSE, expressed as a number in [0,1]).
*/
public static double getRSE(final int k, final double rank, final boolean hra, final long totalN) {
- return getRankUB(k, 2, rank, 1, hra, totalN); //more conservative to assume > 1 level
+ return getRankUB(k, 2, rank, 1, hra, totalN) - rank; //more conservative to assume > 1 level
}
@Override
@@ -188,9 +188,8 @@ abstract class BaseReqSketch implements QuantilesFloatsAPI {
*/
public abstract String viewCompactorDetail(String fmt, boolean allData);
- static boolean exactRank(final int k, final int levels, final double rank,
- final boolean hra, final long totalN) {
- final int baseCap = k * INIT_NUMBER_OF_SECTIONS;
+ static boolean exactRank(final int k, final int levels, final double rank, final boolean hra, final long totalN) {
+ final long baseCap = (long)k * INIT_NUMBER_OF_SECTIONS;
if ((levels == 1) || (totalN <= baseCap)) { return true; }
final double exactRankThresh = (double)baseCap / totalN;
return (hra ? (rank >= (1.0 - exactRankThresh)) : (rank <= exactRankThresh));
diff --git a/tools/SketchesCheckstyle.xml b/tools/SketchesCheckstyle.xml
index f51987dc5..0ced9d3e3 100644
--- a/tools/SketchesCheckstyle.xml
+++ b/tools/SketchesCheckstyle.xml
@@ -1,4 +1,7 @@
<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE module PUBLIC
+ "-//Checkstyle//DTD Checkstyle Configuration 1.3//EN"
+ "https://checkstyle.org/dtds/configuration_1_3.dtd">
<!--
Licensed to the Apache Software Foundation (ASF) under one
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org
For additional commands, e-mail: commits-help@datasketches.apache.org