You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datasketches.apache.org by le...@apache.org on 2020/10/16 21:19:12 UTC
[incubator-datasketches-characterization] branch master updated:
Update char tests for Req, and minor changes to Kll tests.
This is an automated email from the ASF dual-hosted git repository.
leerho pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-datasketches-characterization.git
The following commit(s) were added to refs/heads/master by this push:
new be3b64e Update char tests for Req, and minor changes to Kll tests.
be3b64e is described below
commit be3b64efd11baa631a67fa7a327eab46b168a5f9
Author: Lee Rhodes <le...@users.noreply.github.com>
AuthorDate: Fri Oct 16 14:18:55 2020 -0700
Update char tests for Req, and minor changes to Kll tests.
---
...tchSizeSpeedProfile20200929_115151GMT-07:00.tsv | 0
...etchAccuracyProfile20201015_180629GMT-07:00.tsv | 0
...etchAccuracyProfile20201015_180802GMT-07:00.tsv | 101 +++++++++
.../uniquecount/BaseUpdateSpeedProfile.java | 2 +
...hAccuracyJob.conf => KllSketchAccuracyJob.conf} | 21 +-
.../resources/quantiles/KllSketchSizeSpeedJob.conf | 38 ++++
...AccuracyJob.conf => ReqSketchAccuracy2Job.conf} | 31 ++-
...hAccuracyJob.conf => ReqSketchAccuracyJob.conf} | 14 +-
...AccuracyJob.conf => ReqSketchSizeSpeedJob.conf} | 37 ++--
.../apache/datasketches/MonotonicPointsTest.java | 14 ++
.../characterization/quantiles/FlipFlopStream.java | 102 +++++++++
...yProfile.java => KllSketchAccuracyProfile.java} | 79 ++-----
.../quantiles/KllSketchSizeSpeedProfile.java | 170 ++++++++++++++
.../quantiles/ReqSketchAccuracyProfile.java | 81 ++++---
.../quantiles/ReqSketchAccuracyProfile2.java | 245 +++++++++++++++++++++
.../quantiles/ReqSketchSizeSpeedProfile.java | 178 +++++++++++++++
.../characterization/quantiles/StreamMaker.java | 146 ++++++++++++
.../characterization/quantiles/TrueRanks.java | 172 +++++++++++++++
18 files changed, 1282 insertions(+), 149 deletions(-)
diff --git a/KllSketchSizeSpeedProfile20200929_115151GMT-07:00.tsv b/KllSketchSizeSpeedProfile20200929_115151GMT-07:00.tsv
new file mode 100644
index 0000000..e69de29
diff --git a/ReqSketchAccuracyProfile20201015_180629GMT-07:00.tsv b/ReqSketchAccuracyProfile20201015_180629GMT-07:00.tsv
new file mode 100644
index 0000000..e69de29
diff --git a/ReqSketchAccuracyProfile20201015_180802GMT-07:00.tsv b/ReqSketchAccuracyProfile20201015_180802GMT-07:00.tsv
new file mode 100644
index 0000000..75a3848
--- /dev/null
+++ b/ReqSketchAccuracyProfile20201015_180802GMT-07:00.tsv
@@ -0,0 +1,101 @@
+nPP Value Rank -3SD -2SD -1SD Med +1SD +2SD +3SD 1LB 1UB U
+ 0.0100000000 1 0.0000009537 0.0000000000 0.0000000000 0.0000000000 0.0000000000 0.0000000000 0.0000000000 0.0000000000 0.0000000000 0.0000000000 1
+ 0.0200000000 10593 0.0101022720 -0.0000753403 -0.0000495911 -0.0000257492 0.0000000000 0.0000238419 0.0000495911 0.0000753403 -0.0000263951 0.0000263951 157
+ 0.0300000000 21184 0.0202026367 -0.0001678467 -0.0001134872 -0.0000572205 0.0000019073 0.0000600815 0.0001125336 0.0001668930 -0.0000527852 0.0000527852 323
+ 0.0400000000 31776 0.0303039551 -0.0002250671 -0.0001621246 -0.0000753403 0.0000019073 0.0000839233 0.0001668930 0.0002470016 -0.0000791778 0.0000791778 447
+ 0.0500000000 42368 0.0404052734 -0.0003385544 -0.0002155304 -0.0001058578 0.0000000000 0.0001125336 0.0002260208 0.0003299713 -0.0001055705 0.0001055705 575
+ 0.0600000000 52959 0.0505056381 -0.0004119873 -0.0002603531 -0.0001306534 0.0000009537 0.0001363754 0.0002803802 0.0003957748 -0.0001319606 0.0001319606 683
+ 0.0700000000 63551 0.0606069565 -0.0004701614 -0.0003118515 -0.0001544952 0.0000019073 0.0001668930 0.0003194809 0.0005187988 -0.0001583532 0.0001583532 774
+ 0.0800000000 74143 0.0707082748 -0.0005569458 -0.0003747940 -0.0001840591 0.0000038147 0.0001811981 0.0003738403 0.0005359650 -0.0001847458 0.0001847458 876
+ 0.0900000000 84734 0.0808086395 -0.0006151199 -0.0004310608 -0.0002107620 0.0000076294 0.0002164841 0.0004262924 0.0006141663 -0.0002111359 0.0002111359 980
+ 0.1000000000 95326 0.0909099579 -0.0007505417 -0.0004987717 -0.0002622604 0.0000047684 0.0002651215 0.0005130768 0.0007600784 -0.0002375285 0.0002375285 1117
+ 0.1100000000 105918 0.1010112762 -0.0008420944 -0.0005598068 -0.0002956390 0.0000009537 0.0002851486 0.0005741119 0.0009031296 -0.0002639212 0.0002639212 1214
+ 0.1200000000 116509 0.1111116409 -0.0008878708 -0.0006446838 -0.0003213882 0.0000000000 0.0003366470 0.0006694794 0.0009956360 -0.0002903113 0.0002903113 1336
+ 0.1300000000 127101 0.1212129593 -0.0010671616 -0.0006504059 -0.0003318787 0.0000076294 0.0003433228 0.0006723404 0.0010585785 -0.0003167039 0.0003167039 1376
+ 0.1400000000 137693 0.1313142776 -0.0010280609 -0.0007343292 -0.0003595352 0.0000019073 0.0003814697 0.0007343292 0.0010890961 -0.0003430965 0.0003430965 1470
+ 0.1500000000 148284 0.1414146423 -0.0010976791 -0.0007400513 -0.0003643036 0.0000066757 0.0003986359 0.0007743835 0.0011396408 -0.0003694866 0.0003694866 1471
+ 0.1600000000 158876 0.1515159607 -0.0012865067 -0.0008125305 -0.0003986359 0.0000133514 0.0004320145 0.0008401871 0.0011882782 -0.0003958792 0.0003958792 1611
+ 0.1700000000 169468 0.1616172791 -0.0013608932 -0.0008525848 -0.0004062653 0.0000019073 0.0004482269 0.0008659363 0.0012674332 -0.0004222719 0.0004222719 1606
+ 0.1800000000 180059 0.1717176437 -0.0013246536 -0.0008792877 -0.0004539490 0.0000095367 0.0004644394 0.0008916855 0.0014505386 -0.0004486620 0.0004486620 1663
+ 0.1900000000 190651 0.1818189621 -0.0013713837 -0.0009517670 -0.0004520416 0.0000066757 0.0004940033 0.0009441376 0.0014085770 -0.0004750546 0.0004750546 1717
+ 0.2000000000 201243 0.1919202805 -0.0013380051 -0.0009107590 -0.0004529953 0.0000057220 0.0004968643 0.0009841919 0.0014181137 -0.0005014472 0.0005014472 1677
+ 0.2100000000 211834 0.2020206451 -0.0013475418 -0.0009613037 -0.0004720688 -0.0000019073 0.0004892349 0.0009593964 0.0014410019 -0.0005278373 0.0005278373 1689
+ 0.2200000000 222426 0.2121219635 -0.0013647079 -0.0009317398 -0.0004911423 0.0000009537 0.0004768372 0.0009860992 0.0014543533 -0.0005542299 0.0005542299 1679
+ 0.2300000000 233018 0.2222232819 -0.0014381409 -0.0010509491 -0.0005607605 -0.0000009537 0.0005512238 0.0011014938 0.0015983582 -0.0005806226 0.0005806226 1820
+ 0.2400000000 243609 0.2323236465 -0.0015563965 -0.0010881424 -0.0005626678 0.0000085831 0.0005712509 0.0011091232 0.0016765594 -0.0006070127 0.0006070127 1886
+ 0.2500000000 254201 0.2424249649 -0.0016355515 -0.0011262894 -0.0005970001 -0.0000123978 0.0005836487 0.0011863708 0.0017824173 -0.0006334053 0.0006334053 1904
+ 0.2600000000 264793 0.2525262833 -0.0016899109 -0.0011730194 -0.0006093979 -0.0000114441 0.0005817413 0.0011634827 0.0018281937 -0.0006597979 0.0006597979 1927
+ 0.2700000000 275384 0.2626266479 -0.0017824173 -0.0011892319 -0.0005865097 -0.0000028610 0.0006179810 0.0012149811 0.0019531250 -0.0006861880 0.0006861880 1950
+ 0.2800000000 285976 0.2727279663 -0.0017061234 -0.0011634827 -0.0006036758 0.0000104904 0.0006198883 0.0012369156 0.0018644333 -0.0007125806 0.0007125806 1934
+ 0.2900000000 296568 0.2828292847 -0.0017910004 -0.0012397766 -0.0006418228 -0.0000076294 0.0006093979 0.0011978149 0.0018558502 -0.0007389733 0.0007389733 1975
+ 0.3000000000 307159 0.2929296494 -0.0018682480 -0.0012092590 -0.0005989075 -0.0000019073 0.0006408691 0.0012416840 0.0017833710 -0.0007653634 0.0007653634 1951
+ 0.3100000000 317751 0.3030309677 -0.0018186569 -0.0011978149 -0.0006303787 -0.0000104904 0.0006275177 0.0013170242 0.0019407272 -0.0007917560 0.0007917560 2003
+ 0.3200000000 328343 0.3131322861 -0.0019865036 -0.0012874603 -0.0006484985 -0.0000019073 0.0006542206 0.0012903214 0.0019330978 -0.0008181486 0.0008181486 2017
+ 0.3300000000 338934 0.3232326508 -0.0019016266 -0.0013437271 -0.0006828308 0.0000076294 0.0006647110 0.0013132095 0.0020294189 -0.0008445387 0.0008445387 2059
+ 0.3400000000 349526 0.3333339691 -0.0019350052 -0.0013370514 -0.0006885529 -0.0000104904 0.0006818771 0.0013570786 0.0020389557 -0.0008709313 0.0008709313 2093
+ 0.3500000000 360118 0.3434352875 -0.0020971298 -0.0013771057 -0.0006637573 -0.0000114441 0.0007047653 0.0013923645 0.0020122528 -0.0008973240 0.0008973240 2120
+ 0.3600000000 370709 0.3535356522 -0.0022888184 -0.0015745163 -0.0007686615 0.0000543594 0.0007762909 0.0015277863 0.0023908615 -0.0009237141 0.0009237141 2182
+ 0.3700000000 381301 0.3636369705 -0.0023107529 -0.0014991760 -0.0007905960 0.0000228882 0.0008106232 0.0016012192 0.0023536682 -0.0009501067 0.0009501067 2259
+ 0.3800000000 391893 0.3737382889 -0.0023403168 -0.0015926361 -0.0007724762 0.0000200272 0.0008230209 0.0016098022 0.0021905899 -0.0009764993 0.0009764993 2283
+ 0.3900000000 402484 0.3838386536 -0.0023441315 -0.0015563965 -0.0008106232 0.0000467300 0.0008039474 0.0016412735 0.0025234222 -0.0010028894 0.0010028894 2301
+ 0.4000000000 413076 0.3939399719 -0.0021772385 -0.0015220642 -0.0007934570 0.0000238419 0.0007991791 0.0016241074 0.0022792816 -0.0010292820 0.0010292820 2240
+ 0.4100000000 423668 0.4040412903 -0.0022611618 -0.0015325546 -0.0008344650 0.0000257492 0.0007972717 0.0015115738 0.0022306442 -0.0010556747 0.0010556747 2213
+ 0.4200000000 434259 0.4141416550 -0.0023488998 -0.0015430450 -0.0008230209 0.0000104904 0.0008220673 0.0015287399 0.0022363663 -0.0010820648 0.0010820648 2307
+ 0.4300000000 444851 0.4242429733 -0.0021839142 -0.0015287399 -0.0007963181 0.0000123978 0.0008497238 0.0016670227 0.0023078918 -0.0011084574 0.0011084574 2325
+ 0.4400000000 455443 0.4343442917 -0.0025243759 -0.0015592575 -0.0007829666 0.0000085831 0.0008487701 0.0015964508 0.0024461746 -0.0011348500 0.0011348500 2343
+ 0.4500000000 466034 0.4444446564 -0.0021667480 -0.0015630722 -0.0007867813 0.0000057220 0.0008602142 0.0016269684 0.0023508072 -0.0011612401 0.0011612401 2292
+ 0.4600000000 476626 0.4545459747 -0.0024347305 -0.0015563965 -0.0007705688 -0.0000123978 0.0008354187 0.0016031265 0.0022211075 -0.0011876327 0.0011876327 2293
+ 0.4700000000 487218 0.4646472931 -0.0025815964 -0.0015792847 -0.0008020401 -0.0000276566 0.0007877350 0.0015344620 0.0022583008 -0.0012000000 0.0012000000 2310
+ 0.4800000000 497809 0.4747476578 -0.0024290085 -0.0016431808 -0.0008316040 -0.0000600815 0.0008211136 0.0017290115 0.0024728775 -0.0012000000 0.0012000000 2352
+ 0.4900000000 508401 0.4848489761 -0.0026121140 -0.0017194748 -0.0008897781 0.0000181198 0.0009317398 0.0017547607 0.0026702881 -0.0012000000 0.0012000000 2413
+ 0.5000000000 518993 0.4949502945 -0.0024757385 -0.0017881393 -0.0008773804 -0.0000190735 0.0009050369 0.0018634796 0.0026206970 -0.0012000000 0.0012000000 2436
+ 0.5100000000 529584 0.5050506592 -0.0026559830 -0.0017099380 -0.0008516312 -0.0000019073 0.0009241104 0.0018625259 0.0026092529 -0.0012000000 0.0012000000 2463
+ 0.5200000000 540176 0.5151519775 -0.0024824142 -0.0017385483 -0.0008926392 0.0000209808 0.0008869171 0.0017423630 0.0026817322 -0.0012000000 0.0012000000 2427
+ 0.5300000000 550768 0.5252532959 -0.0026636124 -0.0017910004 -0.0008869171 -0.0000038147 0.0009479523 0.0018558502 0.0027999878 -0.0012000000 0.0012000000 2483
+ 0.5400000000 561359 0.5353536606 -0.0029363632 -0.0018301010 -0.0009260178 0.0000095367 0.0008926392 0.0018148422 0.0027704239 -0.0012000000 0.0012000000 2423
+ 0.5500000000 571951 0.5454549789 -0.0025777817 -0.0017671585 -0.0008964539 0.0000114441 0.0008716583 0.0017576218 0.0027484894 -0.0012000000 0.0012000000 2373
+ 0.5600000000 582543 0.5555562973 -0.0026884079 -0.0017175674 -0.0008459091 0.0000333786 0.0008783340 0.0017719269 0.0027713776 -0.0012000000 0.0012000000 2376
+ 0.5700000000 593134 0.5656566620 -0.0026464462 -0.0017910004 -0.0008964539 0.0000343323 0.0009250641 0.0017967224 0.0026435852 -0.0012000000 0.0012000000 2413
+ 0.5800000000 603726 0.5757579803 -0.0025796890 -0.0017805099 -0.0008792877 0.0000247955 0.0008859634 0.0017967224 0.0027923584 -0.0012000000 0.0012000000 2252
+ 0.5900000000 614318 0.5858592987 -0.0025110245 -0.0018844604 -0.0009279251 0.0000381470 0.0009088516 0.0017337799 0.0025186539 -0.0012000000 0.0012000000 2157
+ 0.6000000000 624909 0.5959596634 -0.0027236938 -0.0018301010 -0.0008745193 -0.0000085831 0.0008592606 0.0017156601 0.0025835037 -0.0012000000 0.0012000000 2001
+ 0.6100000000 635501 0.6060609818 -0.0027532578 -0.0017757416 -0.0009222031 0.0000352859 0.0009088516 0.0018339157 0.0026197433 -0.0012000000 0.0012000000 1992
+ 0.6200000000 646093 0.6161623001 -0.0026407242 -0.0017843246 -0.0009012222 -0.0000047684 0.0009298325 0.0017919540 0.0025215149 -0.0012000000 0.0012000000 2025
+ 0.6300000000 656684 0.6262626648 -0.0026865005 -0.0017108917 -0.0008935928 -0.0000066757 0.0008850098 0.0018348694 0.0025501251 -0.0012000000 0.0012000000 1973
+ 0.6400000000 667276 0.6363639832 -0.0026960373 -0.0016841888 -0.0009222031 -0.0000524521 0.0009050369 0.0017414093 0.0027265549 -0.0012000000 0.0012000000 1965
+ 0.6500000000 677868 0.6464653015 -0.0026159286 -0.0017070770 -0.0008764267 -0.0000057220 0.0009422302 0.0018005371 0.0026721954 -0.0012000000 0.0012000000 1918
+ 0.6600000000 688459 0.6565656662 -0.0026712418 -0.0018005371 -0.0008583069 0.0000095367 0.0008907318 0.0018177032 0.0025796890 -0.0012000000 0.0012000000 1936
+ 0.6700000000 699051 0.6666669846 -0.0030174255 -0.0024833679 -0.0008068085 -0.0001726151 0.0014085770 0.0021553040 0.0036058426 -0.0012000000 0.0012000000 2021
+ 0.6800000000 709643 0.6767683029 -0.0029497147 -0.0024290085 -0.0007648468 -0.0001697540 0.0014257431 0.0021085739 0.0028028488 -0.0012000000 0.0012000000 1945
+ 0.6900000000 720234 0.6868686676 -0.0030326843 -0.0024118423 -0.0008249283 -0.0002031326 0.0013999939 0.0021038055 0.0027675629 -0.0012000000 0.0012000000 1995
+ 0.7000000000 730826 0.6969699860 -0.0031080246 -0.0024814606 -0.0008878708 -0.0001773834 0.0014419556 0.0020389557 0.0037240982 -0.0012000000 0.0012000000 1985
+ 0.7100000000 741418 0.7070713043 -0.0030717850 -0.0024528503 -0.0008554459 -0.0002460480 0.0013761520 0.0020523071 0.0033168793 -0.0012000000 0.0012000000 1906
+ 0.7200000000 752009 0.7171716690 -0.0031852722 -0.0025110245 -0.0009069443 -0.0003013611 0.0013952255 0.0020122528 0.0035734177 -0.0012000000 0.0012000000 1914
+ 0.7300000000 762601 0.7272729874 -0.0031871796 -0.0024805069 -0.0009021759 -0.0002822876 0.0013217926 0.0019445419 0.0034885406 -0.0012000000 0.0012000000 1915
+ 0.7400000000 773193 0.7373743057 -0.0031614304 -0.0024385452 -0.0009651184 -0.0003261566 0.0013561249 0.0019702911 0.0034246445 -0.0012000000 0.0012000000 1970
+ 0.7500000000 783784 0.7474746704 -0.0033473969 -0.0025119781 -0.0010175705 -0.0002975464 0.0013084412 0.0019044876 0.0034818649 -0.0012000000 0.0012000000 1892
+ 0.7600000000 794376 0.7575759888 -0.0034284592 -0.0024509430 -0.0009918213 -0.0002689362 0.0013103485 0.0018386841 0.0035314560 -0.0012000000 0.0012000000 1864
+ 0.7700000000 804968 0.7676773071 -0.0033655167 -0.0022640228 -0.0010519028 -0.0003299713 0.0012588501 0.0018539429 0.0034389496 -0.0012000000 0.0012000000 1822
+ 0.7800000000 815559 0.7777776718 -0.0033588409 -0.0022478104 -0.0011110306 -0.0002746582 0.0012083054 0.0018129349 0.0032701492 -0.0012000000 0.0012000000 1856
+ 0.7900000000 826151 0.7878789902 -0.0032835007 -0.0019512177 -0.0010986328 -0.0002412796 0.0012245178 0.0018367767 0.0034141541 -0.0012000000 0.0012000000 1859
+ 0.8000000000 836743 0.7979803085 -0.0033378601 -0.0018882751 -0.0011501312 -0.0002212524 0.0011558533 0.0018377304 0.0034475327 -0.0012000000 0.0012000000 1788
+ 0.8100000000 847334 0.8080806732 -0.0034370422 -0.0018472672 -0.0011329651 0.0000000000 0.0010976791 0.0017948151 0.0036478043 -0.0012000000 0.0012000000 1708
+ 0.8200000000 857926 0.8181819916 -0.0036230087 -0.0019035339 -0.0012073517 0.0002698898 0.0011234283 0.0020818710 0.0034389496 -0.0012000000 0.0012000000 1793
+ 0.8300000000 868518 0.8282833099 -0.0032329559 -0.0017871857 -0.0012617111 0.0003108978 0.0010719299 0.0021553040 0.0033712387 -0.0012000000 0.0012000000 1774
+ 0.8400000000 879109 0.8383836746 -0.0034103394 -0.0018396378 -0.0012292862 0.0003614426 0.0010814667 0.0021877289 0.0033025742 -0.0012000000 0.0012000000 1764
+ 0.8500000000 889701 0.8484849930 -0.0034685135 -0.0018959045 -0.0012884140 0.0003814697 0.0010204315 0.0023784637 0.0032253265 -0.0012000000 0.0012000000 1663
+ 0.8600000000 900293 0.8585863113 -0.0033397675 -0.0018653870 -0.0012779236 0.0003318787 0.0009584427 0.0023794174 0.0032587051 -0.0012000000 0.0012000000 1691
+ 0.8700000000 910884 0.8686866760 -0.0032901764 -0.0019330978 -0.0013332367 0.0002851486 0.0009880066 0.0024414063 0.0034379959 -0.0012000000 0.0012000000 1698
+ 0.8800000000 921476 0.8787879944 -0.0034999847 -0.0019016266 -0.0012969971 0.0003061295 0.0009222031 0.0024023056 0.0031290054 -0.0012000000 0.0012000000 1633
+ 0.8900000000 932068 0.8888893127 -0.0032014847 -0.0019741058 -0.0013494492 0.0003404617 0.0008687973 0.0024385452 0.0032720566 -0.0012000000 0.0012000000 1582
+ 0.9000000000 942659 0.8989896774 -0.0037488937 -0.0020399094 -0.0014276505 0.0002708435 0.0008754730 0.0025758743 0.0032062531 -0.0012000000 0.0012000000 1614
+ 0.9100000000 953251 0.9090909958 -0.0033588409 -0.0020027161 -0.0014028549 0.0002107620 0.0008363724 0.0024156570 0.0031442642 -0.0012000000 0.0012000000 1600
+ 0.9200000000 963843 0.9191923141 -0.0030479431 -0.0020656586 -0.0014820099 0.0002336502 0.0008420944 0.0024671555 0.0030546188 -0.0012000000 0.0012000000 1495
+ 0.9300000000 974434 0.9292926788 -0.0036220551 -0.0021476746 -0.0014238358 0.0001707077 0.0007848740 0.0024690628 0.0030746460 -0.0012000000 0.0012000000 1548
+ 0.9400000000 985026 0.9393939972 -0.0033369064 -0.0022249222 -0.0013942719 0.0002059937 0.0008106232 0.0025148392 0.0029973984 -0.0012000000 0.0012000000 1498
+ 0.9500000000 995618 0.9494953156 -0.0026865005 -0.0020885468 -0.0014610291 0.0001382828 0.0007486343 0.0024480820 0.0030632019 -0.0012000000 0.0012000000 1396
+ 0.9600000000 1006209 0.9595956802 -0.0025234222 -0.0021381378 -0.0014200211 0.0000762939 0.0007638931 0.0022850037 0.0028791428 -0.0012000000 0.0012000000 1321
+ 0.9700000000 1016801 0.9696969986 -0.0030612946 -0.0023288727 -0.0013608932 0.0001029968 0.0007171631 0.0024213791 0.0029067993 -0.0012000000 0.0012000000 1277
+ 0.9800000000 1027393 0.9797983170 -0.0026569366 -0.0022916794 -0.0013227463 0.0000371933 0.0007495880 0.0023622513 0.0028581619 -0.0012000000 0.0012000000 1080
+ 0.9900000000 1037984 0.9898986816 -0.0028610229 -0.0022287369 -0.0009107590 0.0000686646 0.0008163452 0.0022869110 0.0027770996 -0.0012000000 0.0012000000 762
+ 1.0000000000 1048576 1.0000000000 0.0000000000 0.0000000000 0.0000000000 0.0000000000 0.0000000000 0.0000000000 0.0000000000 -0.0012000000 0.0012000000 1
diff --git a/src/main/java/org/apache/datasketches/characterization/uniquecount/BaseUpdateSpeedProfile.java b/src/main/java/org/apache/datasketches/characterization/uniquecount/BaseUpdateSpeedProfile.java
index 703b255..fd123ac 100644
--- a/src/main/java/org/apache/datasketches/characterization/uniquecount/BaseUpdateSpeedProfile.java
+++ b/src/main/java/org/apache/datasketches/characterization/uniquecount/BaseUpdateSpeedProfile.java
@@ -102,7 +102,9 @@ public abstract class BaseUpdateSpeedProfile implements JobProfile {
sumUpdateTimePerU_nS += doTrial(nextU);
}
final double meanUpdateTimePerU_nS = sumUpdateTimePerU_nS / trials;
+
process(meanUpdateTimePerU_nS, trials, nextU, dataStr);
+
job.println(dataStr.toString());
}
}
diff --git a/src/main/resources/quantiles/ReqFloatsSketchAccuracyJob.conf b/src/main/resources/quantiles/KllSketchAccuracyJob.conf
similarity index 59%
copy from src/main/resources/quantiles/ReqFloatsSketchAccuracyJob.conf
copy to src/main/resources/quantiles/KllSketchAccuracyJob.conf
index cb76163..347725f 100644
--- a/src/main/resources/quantiles/ReqFloatsSketchAccuracyJob.conf
+++ b/src/main/resources/quantiles/KllSketchAccuracyJob.conf
@@ -15,35 +15,24 @@
# specific language governing permissions and limitations
# under the License.
-JobProfile=org.apache.datasketches.characterization.quantiles.ReqSketchAccuracyProfile
+JobProfile=org.apache.datasketches.characterization.quantiles.KllSketchAccuracyProfile
# Trials config (indep of sketch)
## Stream lengths
-LgMin=17 # The starting stream length
-LgMax=17 # How high the stream length goes
+LgMin=11 # The starting stream length
+LgMax=11 # How high the stream length goes
LgDelta=3 # If > 0, this is the lg Increment
PPO=8 # The horizontal x-resolution of trials points
-LgTrials=12 # lgTrials at every stream length
+LgTrials=14 # lgTrials at every stream length
ErrSkLgK=12 # parameter for the rank error distribution sketch
# Plotting
NumPlotPoints=100 # number of plot points along the x-axis
-EvenlySpaced=true # if true the x-axis points will be evenly spaced ranks in [0,1], otherwise exponential in [0,1]
-Exponent=2.0 # the steepness of the exponential x-axis density gradient curve, must be >= 1.0
-StdDev=1 # std deviation used when plotting LB, UB
-RankRange=1.0 #.0002 # range of rank to plot. If LRA => 0 to 0.3; if HRA => 0.7 to 1.0
# Specific sketch config
-K=50 # sketch size and accuracy parameter
+K=100 # sketch size and accuracy parameter
-#For LRA, LE,GT have the converged point at rank 1.0
-#For HRA, LT,GE have the converged point at rank 0.0
-Criterion=LE # LT, LE, GT, GE, must be all caps
-HRA=false # if true use high-rank accuracy, otherwise low-rank accuracy
-
-#ReqDebugLevel=2 # or 0, 1, 2. disable by commenting it out
-#ReqDebugFmt=%5.0f
# Date-Time Profile
TimeZone=PDT
diff --git a/src/main/resources/quantiles/KllSketchSizeSpeedJob.conf b/src/main/resources/quantiles/KllSketchSizeSpeedJob.conf
new file mode 100644
index 0000000..d5e759a
--- /dev/null
+++ b/src/main/resources/quantiles/KllSketchSizeSpeedJob.conf
@@ -0,0 +1,38 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+JobProfile=org.apache.datasketches.characterization.quantiles.KllSketchSizeSpeedProfile
+
+# Trials config (indep of sketch)
+LgMinT=4
+LgMaxT=12
+
+## Stream lengths
+LgMinSL=1 # The starting stream length
+LgMaxSL=20 # How high the stream length goes
+PpoSL=2 # The horizontal x-resolution of trials points per octave
+LgMinBpSL=10 # The SL breakpoint where the slope starts
+LgMaxBpSL=20 # The SL breakpoint where the slope stops
+
+# Specific sketch config
+KllK=100 # KllSketch size and accuracy
+
+# Date-Time Profile
+TimeZone=PDT
+TimeZoneOffset=-25200000 # offset in millisec: PST (UTC-8) = -28_800_000 PDT (UTC-7) = -25_200_000
+FileNameDateFormat=yyyyMMdd'_'HHmmssz
+ReadableDateFormat=yyyy/MM/dd HH:mm:ss
\ No newline at end of file
diff --git a/src/main/resources/quantiles/ReqFloatsSketchAccuracyJob.conf b/src/main/resources/quantiles/ReqSketchAccuracy2Job.conf
similarity index 62%
copy from src/main/resources/quantiles/ReqFloatsSketchAccuracyJob.conf
copy to src/main/resources/quantiles/ReqSketchAccuracy2Job.conf
index cb76163..22e0753 100644
--- a/src/main/resources/quantiles/ReqFloatsSketchAccuracyJob.conf
+++ b/src/main/resources/quantiles/ReqSketchAccuracy2Job.conf
@@ -15,34 +15,31 @@
# specific language governing permissions and limitations
# under the License.
-JobProfile=org.apache.datasketches.characterization.quantiles.ReqSketchAccuracyProfile
+JobProfile=org.apache.datasketches.characterization.quantiles.ReqSketchAccuracyProfile2
# Trials config (indep of sketch)
## Stream lengths
-LgMin=17 # The starting stream length
-LgMax=17 # How high the stream length goes
-LgDelta=3 # If > 0, this is the lg Increment
-PPO=8 # The horizontal x-resolution of trials points
-
-LgTrials=12 # lgTrials at every stream length
-ErrSkLgK=12 # parameter for the rank error distribution sketch
-
+LgSL=20 # The stream length
# Plotting
NumPlotPoints=100 # number of plot points along the x-axis
-EvenlySpaced=true # if true the x-axis points will be evenly spaced ranks in [0,1], otherwise exponential in [0,1]
-Exponent=2.0 # the steepness of the exponential x-axis density gradient curve, must be >= 1.0
+
StdDev=1 # std deviation used when plotting LB, UB
-RankRange=1.0 #.0002 # range of rank to plot. If LRA => 0 to 0.3; if HRA => 0.7 to 1.0
-# Specific sketch config
-K=50 # sketch size and accuracy parameter
+# Patterns
+Pattern=Random
+Offset=1 #0 for min value of 0; 1 for min value of 1
+
+ErrQSkLgK=12 # the rank error distribution sketch LgK
+# Target sketch config
+K=50 # sketch size and accuracy parameter
+HRA=true # if true use high-rank accuracy, otherwise low-rank accuracy
#For LRA, LE,GT have the converged point at rank 1.0
#For HRA, LT,GE have the converged point at rank 0.0
-Criterion=LE # LT, LE, GT, GE, must be all caps
-HRA=false # if true use high-rank accuracy, otherwise low-rank accuracy
+Criterion=LE # LT, LE Must be all caps.
+Compatible=false
-#ReqDebugLevel=2 # or 0, 1, 2. disable by commenting it out
+#ReqDebugLevel=2 # or 0, 1, 2. disable by commenting it out. Use only when LgTrials=0
#ReqDebugFmt=%5.0f
# Date-Time Profile
diff --git a/src/main/resources/quantiles/ReqFloatsSketchAccuracyJob.conf b/src/main/resources/quantiles/ReqSketchAccuracyJob.conf
similarity index 81%
copy from src/main/resources/quantiles/ReqFloatsSketchAccuracyJob.conf
copy to src/main/resources/quantiles/ReqSketchAccuracyJob.conf
index cb76163..76e9d03 100644
--- a/src/main/resources/quantiles/ReqFloatsSketchAccuracyJob.conf
+++ b/src/main/resources/quantiles/ReqSketchAccuracyJob.conf
@@ -19,30 +19,32 @@ JobProfile=org.apache.datasketches.characterization.quantiles.ReqSketchAccuracyP
# Trials config (indep of sketch)
## Stream lengths
-LgMin=17 # The starting stream length
-LgMax=17 # How high the stream length goes
+LgMin=20 # The starting stream length
+LgMax=20 # How high the stream length goes
LgDelta=3 # If > 0, this is the lg Increment
PPO=8 # The horizontal x-resolution of trials points
LgTrials=12 # lgTrials at every stream length
-ErrSkLgK=12 # parameter for the rank error distribution sketch
+ErrQSkLgK=12 # the rank error distribution sketch LgK
+ErrHllSkLgK=12 # the rank error HLL sketch Lgk
# Plotting
NumPlotPoints=100 # number of plot points along the x-axis
EvenlySpaced=true # if true the x-axis points will be evenly spaced ranks in [0,1], otherwise exponential in [0,1]
Exponent=2.0 # the steepness of the exponential x-axis density gradient curve, must be >= 1.0
StdDev=1 # std deviation used when plotting LB, UB
-RankRange=1.0 #.0002 # range of rank to plot. If LRA => 0 to 0.3; if HRA => 0.7 to 1.0
+RankRange=1.0 # range of rank to plot. E.g., given 0.3: if LRA => 0 to 0.3; if HRA => 0.7 to 1.0
# Specific sketch config
K=50 # sketch size and accuracy parameter
#For LRA, LE,GT have the converged point at rank 1.0
#For HRA, LT,GE have the converged point at rank 0.0
-Criterion=LE # LT, LE, GT, GE, must be all caps
+Criterion=LE # LT, LE, GT, GE. Must be all caps.
+Compatible=false
HRA=false # if true use high-rank accuracy, otherwise low-rank accuracy
-#ReqDebugLevel=2 # or 0, 1, 2. disable by commenting it out
+#ReqDebugLevel=2 # or 0, 1, 2. disable by commenting it out. Use only when LgTrials=0
#ReqDebugFmt=%5.0f
# Date-Time Profile
diff --git a/src/main/resources/quantiles/ReqFloatsSketchAccuracyJob.conf b/src/main/resources/quantiles/ReqSketchSizeSpeedJob.conf
similarity index 58%
rename from src/main/resources/quantiles/ReqFloatsSketchAccuracyJob.conf
rename to src/main/resources/quantiles/ReqSketchSizeSpeedJob.conf
index cb76163..5b52f2f 100644
--- a/src/main/resources/quantiles/ReqFloatsSketchAccuracyJob.conf
+++ b/src/main/resources/quantiles/ReqSketchSizeSpeedJob.conf
@@ -15,38 +15,31 @@
# specific language governing permissions and limitations
# under the License.
-JobProfile=org.apache.datasketches.characterization.quantiles.ReqSketchAccuracyProfile
+JobProfile=org.apache.datasketches.characterization.quantiles.ReqSketchSizeSpeedProfile
# Trials config (indep of sketch)
-## Stream lengths
-LgMin=17 # The starting stream length
-LgMax=17 # How high the stream length goes
-LgDelta=3 # If > 0, this is the lg Increment
-PPO=8 # The horizontal x-resolution of trials points
-
-LgTrials=12 # lgTrials at every stream length
-ErrSkLgK=12 # parameter for the rank error distribution sketch
+LgMinT=4
+LgMaxT=12
-# Plotting
-NumPlotPoints=100 # number of plot points along the x-axis
-EvenlySpaced=true # if true the x-axis points will be evenly spaced ranks in [0,1], otherwise exponential in [0,1]
-Exponent=2.0 # the steepness of the exponential x-axis density gradient curve, must be >= 1.0
-StdDev=1 # std deviation used when plotting LB, UB
-RankRange=1.0 #.0002 # range of rank to plot. If LRA => 0 to 0.3; if HRA => 0.7 to 1.0
+## Stream lengths
+LgMinSL=1 # The starting stream length
+LgMaxSL=20 # How high the stream length goes
+PpoSL=2 # The horizontal x-resolution of trials points per octave
+LgMinBpSL=10 # The SL breakpoint where the slope starts
+LgMaxBpSL=20 # The SL breakpoint where the slope stops
# Specific sketch config
-K=50 # sketch size and accuracy parameter
-
-#For LRA, LE,GT have the converged point at rank 1.0
-#For HRA, LT,GE have the converged point at rank 0.0
-Criterion=LE # LT, LE, GT, GE, must be all caps
+ReqK=12 # ReqSketch size and accuracy
HRA=false # if true use high-rank accuracy, otherwise low-rank accuracy
+#For HRA, LT,GE have the converged point at rank 0.0
+#For LRA, LE,GT have the converged point at rank 1.0
+Criterion=LE # LT, LE, GT, GE. Must be all caps.
-#ReqDebugLevel=2 # or 0, 1, 2. disable by commenting it out
+#ReqDebugLevel=2 # or 0, 1, 2. disable by commenting it out. Use only when LgTrials=0
#ReqDebugFmt=%5.0f
# Date-Time Profile
TimeZone=PDT
TimeZoneOffset=-25200000 # offset in millisec: PST (UTC-8) = -28_800_000 PDT (UTC-7) = -25_200_000
FileNameDateFormat=yyyyMMdd'_'HHmmssz
-ReadableDateFormat=yyyy/MM/dd HH:mm:ss
+ReadableDateFormat=yyyy/MM/dd HH:mm:ss
\ No newline at end of file
diff --git a/src/test/java/org/apache/datasketches/MonotonicPointsTest.java b/src/test/java/org/apache/datasketches/MonotonicPointsTest.java
index 7ae2675..f56b6ee 100644
--- a/src/test/java/org/apache/datasketches/MonotonicPointsTest.java
+++ b/src/test/java/org/apache/datasketches/MonotonicPointsTest.java
@@ -19,6 +19,8 @@
package org.apache.datasketches;
+import static org.testng.Assert.assertEquals;
+
import org.testng.annotations.Test;
/**
@@ -33,6 +35,18 @@ public class MonotonicPointsTest {
for (int i = 0; i < arr.length; i++) { println(arr[i] + ""); }
}
+ @Test
+ public void checkMonotonicPoints2() {
+ double[] arr = MonotonicPoints.evenlySpaced(0, 1, 3, false);
+ assertEquals(arr[0], 0.0);
+ assertEquals(arr[1], 0.5);
+ assertEquals(arr[2], 1.0);
+ arr = MonotonicPoints.evenlySpaced(3, 7, 3, false);
+ assertEquals(arr[0], 3.0);
+ assertEquals(arr[1], 5.0);
+ assertEquals(arr[2], 7.0);
+ }
+
@Test void checkEvenlySpacedPoints() {
double[] arr = Util.evenlySpaced(0.0, 100.0, 21);
for (int i = 0; i < arr.length; i++) { println(arr[i] + ""); }
diff --git a/src/test/java/org/apache/datasketches/characterization/quantiles/FlipFlopStream.java b/src/test/java/org/apache/datasketches/characterization/quantiles/FlipFlopStream.java
new file mode 100644
index 0000000..d526a52
--- /dev/null
+++ b/src/test/java/org/apache/datasketches/characterization/quantiles/FlipFlopStream.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.datasketches.characterization.quantiles;
+
+import org.testng.annotations.Test;
+
+/**
+ * Generate multiple flip-flop sequences with the given N.
+ * @author Lee Rhodes
+ */
+public class FlipFlopStream {
+ private float[] arr;
+ private int v;
+ private int idx;
+ private int low;
+ private int high;
+ private int lo;
+ private int hi;
+
+ /**
+ * Constuctor used by TestNG
+ */
+ public FlipFlopStream() {}
+
+ /**
+ * Construct an overall sequence of size N
+ * @param N the length of the sequence and size of the returned array.
+ * @param offset The lowest value in the sequence. Usually either 0 or 1.
+ */
+ public FlipFlopStream(int N, int offset) {
+ arr = new float[N];
+ idx = 0;
+ v = offset;
+ low = 0;
+ high = N - 1;
+ lo = low;
+ hi = high;
+ }
+
+ /**
+ * Generates a flip-flop sequence
+ * @param loReps repetitions for the low range
+ * @param hiReps repetitions for the high range
+ * @param steps maximum number of steps for this sequence
+ */
+ public void flipFlop(int loReps, int hiReps, int steps) {
+ int n = hi - lo + 1;
+ while (n > 0 && steps > 0) {
+ int i = loReps;
+ while (n > 0 && steps > 0 && i > 0) {
+ arr[idx++] = lo++ + v;
+ n--;
+ steps--;
+ i--;
+ }
+ int j = hiReps;
+ while (n > 0 && steps > 0 && j > 0) {
+ arr[idx++] = hi-- + v;
+ n--;
+ steps--;
+ j--;
+ }
+ }
+ }
+
+ /**
+ * @return the populated array
+ */
+ public float[] getArray() {
+ return arr;
+ }
+
+ @Test
+ public void checkFlipFlop() {
+ int N = 50;
+ FlipFlopStream ffs = new FlipFlopStream(N, 1);
+ ffs.flipFlop(1, 1, 20);
+ ffs.flipFlop(10, 1, 10);
+ ffs.flipFlop(1, 10, 10);
+ ffs.flipFlop(1, 1, 10);
+ for (int i = 0; i < N; i++) { println(ffs.arr[i]); }
+ }
+
+ static void println(Object o) { System.out.println(o.toString()); }
+}
diff --git a/src/test/java/org/apache/datasketches/characterization/quantiles/ReqSketchAccuracyProfile.java b/src/test/java/org/apache/datasketches/characterization/quantiles/KllSketchAccuracyProfile.java
similarity index 71%
copy from src/test/java/org/apache/datasketches/characterization/quantiles/ReqSketchAccuracyProfile.java
copy to src/test/java/org/apache/datasketches/characterization/quantiles/KllSketchAccuracyProfile.java
index 57867d0..01f3d92 100644
--- a/src/test/java/org/apache/datasketches/characterization/quantiles/ReqSketchAccuracyProfile.java
+++ b/src/test/java/org/apache/datasketches/characterization/quantiles/KllSketchAccuracyProfile.java
@@ -20,29 +20,24 @@
package org.apache.datasketches.characterization.quantiles;
import static java.lang.Math.round;
-import static org.apache.datasketches.ExponentiallySpacedPoints.expSpacedFloats;
import static org.apache.datasketches.GaussianRanks.GAUSSIANS_3SD;
import static org.apache.datasketches.Util.evenlySpacedFloats;
import static org.apache.datasketches.Util.pwr2LawNext;
-import static org.apache.datasketches.req.Criteria.GE;
-import static org.apache.datasketches.req.Criteria.LT;
import org.apache.datasketches.Job;
import org.apache.datasketches.JobProfile;
import org.apache.datasketches.MonotonicPoints;
import org.apache.datasketches.characterization.Shuffle;
+import org.apache.datasketches.kll.KllFloatsSketch;
import org.apache.datasketches.quantiles.DoublesSketch;
import org.apache.datasketches.quantiles.DoublesSketchBuilder;
import org.apache.datasketches.quantiles.UpdateDoublesSketch;
-import org.apache.datasketches.req.Criteria;
-import org.apache.datasketches.req.ReqDebugImpl;
-import org.apache.datasketches.req.ReqSketch;
-import org.apache.datasketches.req.ReqSketchBuilder;
+
/**
* @author Lee Rhodes
*/
-public class ReqSketchAccuracyProfile implements JobProfile {
+public class KllSketchAccuracyProfile implements JobProfile {
private Job job;
//FROM PROPERTIES
@@ -57,19 +52,12 @@ public class ReqSketchAccuracyProfile implements JobProfile {
//plotting & x-axis configuration
private int numPlotPoints;
- private boolean evenlySpaced;
- private double exponent;
- private int sd;
- private double rankRange;
//Target sketch configuration & error analysis
private int K;
- private boolean hra; //high rank accuracy
- private Criteria criterion;
- private org.apache.datasketches.req.ReqDebugImpl reqDebugImpl = null;
//DERIVED globals
- private ReqSketch sk;
+ private KllFloatsSketch sk;
//The array of Gaussian quantiles for +/- StdDev error analysis
private double[] gRanks;
@@ -82,13 +70,12 @@ public class ReqSketchAccuracyProfile implements JobProfile {
private float[] corrTrueValues;
private final String[] columnLabels =
- {"rPP", "Value", "Rank", "-3SD","-2SD", "-1SD", "Med", "+1SD", "+2SD", "+3SD", "RLB", "RUB"};
+ {"nPP", "Value", "Rank", "-3SD","-2SD", "-1SD", "Med", "+1SD", "+2SD", "+3SD"};
private final String sFmt =
- "%3s\t%5s\t%4s\t%4s\t%4s\t%4s\t%5s\t%4s\t%4s\t%4s\t%3s\t%3s\n";
+ "%3s\t%5s\t%4s\t%4s\t%4s\t%4s\t%5s\t%4s\t%4s\t%4s\n";
private final String fFmt =
"%14.10f\t%14.0f\t%14.10f\t" //rPP, Value, Rank
- + "%14.10f\t%14.10f\t%14.10f\t%14.10f\t%14.10f\t%14.10f\t%14.10f\t" //-3sd to +3sd
- + "%14.10f\t%14.10f\n"; //rlb, rub
+ + "%14.10f\t%14.10f\t%14.10f\t%14.10f\t%14.10f\t%14.10f\t%14.10f\t\n"; //-3sd to +3sd
//JobProfile interface
@Override
@@ -117,27 +104,16 @@ public class ReqSketchAccuracyProfile implements JobProfile {
errorSkLgK = Integer.parseInt(job.getProperties().mustGet("ErrSkLgK"));
//plotting & x-axis config
numPlotPoints = Integer.parseInt(job.getProperties().mustGet("NumPlotPoints"));
- evenlySpaced = Boolean.valueOf(job.getProperties().mustGet("EvenlySpaced"));
- exponent = Double.parseDouble(job.getProperties().mustGet("Exponent"));
- sd = Integer.parseInt(job.getProperties().mustGet("StdDev"));
- rankRange = Double.parseDouble(job.getProperties().mustGet("RankRange"));
//Target sketch config
K = Integer.parseInt(job.getProperties().mustGet("K"));
- hra = Boolean.parseBoolean(job.getProperties().mustGet("HRA"));
- criterion = Criteria.valueOf(job.getProperties().mustGet("Criterion"));
- String reqDebugLevel = job.getProperties().get("ReqDebugLevel");
- String reqDebugFmt = job.getProperties().get("ReqDebugFmt");
- if (reqDebugLevel != null) {
- int level = Integer.parseInt(reqDebugLevel);
- reqDebugImpl = new ReqDebugImpl(level, reqDebugFmt);
- }
+
}
void configureCommon() {
configureSketch();
trueValues = new float[numPlotPoints];
corrTrueValues = new float[numPlotPoints];
- trueValueCorrection = criterion == GE || criterion == LT ? 1 : 0;
+ trueValueCorrection = 1; //KLL is always LT
errQSkArr = new UpdateDoublesSketch[numPlotPoints];
//configure the error quantiles array
final DoublesSketchBuilder builder = DoublesSketch.builder().setK(1 << errorSkLgK);
@@ -151,11 +127,7 @@ public class ReqSketchAccuracyProfile implements JobProfile {
}
void configureSketch() {
- final ReqSketchBuilder bldr = ReqSketch.builder();
- bldr.setK(K).setHighRankAccuracy(hra);
- if (reqDebugImpl != null) { bldr.setReqDebug(reqDebugImpl); }
- sk = bldr.build();
- sk.setCriterion(criterion);
+ sk = new KllFloatsSketch(K);
}
private void doJob() {
@@ -191,17 +163,14 @@ public class ReqSketchAccuracyProfile implements JobProfile {
job.printfData(sFmt, (Object[])columnLabels);
//build the stream
- //the values themselves reflect their integer ranks starting with 1.
+ //the values themselves reflect their integer ranks starting with 1 (except for LT)
stream = new float[streamLength];
for (int sl = 1; sl <= streamLength; sl++) { stream[sl - 1] = sl; } //1 to SL
//compute the true values used at the plot points
- final int subStreamLen = (int)Math.round(rankRange * streamLength);
- final float start = hra ? streamLength - subStreamLen : 1.0f;
- final float end = hra ? streamLength : subStreamLen;
- final float[] fltValues = evenlySpaced
- ? evenlySpacedFloats(start, end, numPlotPoints)
- : expSpacedFloats(1.0f, streamLength, numPlotPoints, exponent, hra);
+ final float start = 1.0f;
+ final float end = streamLength;
+ final float[] fltValues = evenlySpacedFloats(start, end, numPlotPoints);
for (int pp = 0; pp < numPlotPoints; pp++) {
trueValues[pp] = round(fltValues[pp]);
@@ -210,6 +179,7 @@ public class ReqSketchAccuracyProfile implements JobProfile {
//Do numTrials for all plotpoints
for (int t = 0; t < numTrials; t++) {
+ sk = new KllFloatsSketch(K);
doTrial(sk, stream, trueValues, corrTrueValues, errQSkArr);
}
@@ -217,8 +187,6 @@ public class ReqSketchAccuracyProfile implements JobProfile {
for (int pp = 0 ; pp < numPlotPoints; pp++) {
final double v = trueValues[pp];
final double tr = v / streamLength; //the true rank
- final double rlb = sk.getRankLowerBound(tr, sd) - tr;
- final double rub = sk.getRankUpperBound(tr, sd) - tr;
//for each of the numErrDistRanks distributions extract the sd quantiles
final double[] errQ = errQSkArr[pp].getQuantiles(gRanks); //get error values at the Gaussian ranks
@@ -226,12 +194,11 @@ public class ReqSketchAccuracyProfile implements JobProfile {
//Plot the row. We ignore quantiles collected at 0 and 1.0.
final double relPP = (double)(pp + 1) / numPlotPoints;
job.printfData(fFmt, relPP, v, tr,
- errQ[0], errQ[1], errQ[2], errQ[3], errQ[4], errQ[5], errQ[6],
- rlb, rub);
+ errQ[0], errQ[1], errQ[2], errQ[3], errQ[4], errQ[5], errQ[6]);
errQSkArr[pp].reset(); //reset the errQSkArr for next streamLength
}
- job.println(LS + "Serialization Bytes: " + sk.getSerializationBytes());
- job.println(sk.viewCompactorDetail("%5.0f", false));
+ job.println(LS + "Serialization Bytes: " + sk.getSerializedSizeBytes());
+
}
/**
@@ -242,17 +209,19 @@ public class ReqSketchAccuracyProfile implements JobProfile {
* @param trueValues the true integer ranks at each of the plot points
* @param errQSkArr the quantile error sketches for each plot point to be updated
*/
- static void doTrial(final ReqSketch sk, final float[] stream, final float[] trueValues,
+ static void doTrial(final KllFloatsSketch sk, final float[] stream, final float[] trueValues,
final float[] corrTrueValues, final UpdateDoublesSketch[] errQSkArr) {
- sk.reset();
Shuffle.shuffle(stream);
final int sl = stream.length;
for (int i = 0; i < sl; i++) {
sk.update(stream[i]);
}
- //get estimated ranks from sketch for all plotpoints, this is a bulk operation
- final double[] estRanks = sk.getRanks(trueValues);
final int numPP = trueValues.length;
+ //get estimated ranks from sketch for all plotpoints
+ final double[] estRanks = new double[numPP];
+ for (int pp = 0; pp < numPP; pp++) {
+ estRanks[pp] = sk.getRank(trueValues[pp]);
+ }
//compute errors for each plotPoint
for (int pp = 0; pp < numPP; pp++) {
final double errorAtPlotPoint = estRanks[pp] - (double)corrTrueValues[pp] / sl;
diff --git a/src/test/java/org/apache/datasketches/characterization/quantiles/KllSketchSizeSpeedProfile.java b/src/test/java/org/apache/datasketches/characterization/quantiles/KllSketchSizeSpeedProfile.java
new file mode 100644
index 0000000..058237b
--- /dev/null
+++ b/src/test/java/org/apache/datasketches/characterization/quantiles/KllSketchSizeSpeedProfile.java
@@ -0,0 +1,170 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.datasketches.characterization.quantiles;
+
+import static java.lang.Math.log;
+import static java.lang.Math.pow;
+import static org.apache.datasketches.Util.pwr2LawNext;
+
+import org.apache.datasketches.Job;
+import org.apache.datasketches.JobProfile;
+import org.apache.datasketches.Properties;
+import org.apache.datasketches.kll.KllFloatsSketch;
+
+
+/**
+ * @author Lee Rhodes
+ */
+public class KllSketchSizeSpeedProfile implements JobProfile {
+ private Job job;
+ private Properties prop;
+
+ //FROM PROPERTIES
+ private int lgMinT;
+ private int lgMaxT;
+ private int lgMinBpSL;
+ private int lgMaxBpSL;
+ //For computing the different stream lengths
+ private int lgMinSL;
+ private int lgMaxSL;
+ private int ppoSL;
+
+ private double slope;
+
+ //Target sketch configuration & error analysis
+ private int kllK;
+
+ //DERIVED & GLOBALS
+ private KllFloatsSketch kllSk;
+
+
+ private final String[] columnLabels = {"PP", "SL", "Trials", "KllBytes", "Kll nS" };
+ private final String sFmt = "%2s\t%2s\t%6s\t%8s\t%4s\n";
+ private final String dFmt = "%,6d\t%,12d\t%,12d\t%,12d\t%,12.6f\n";
+
+ private void extractProperties() {
+ //trials config
+ lgMinT = Integer.parseInt(prop.mustGet("LgMinT"));
+ lgMaxT = Integer.parseInt(prop.mustGet("LgMaxT"));
+ lgMinBpSL = Integer.parseInt(prop.mustGet("LgMinBpSL"));
+ lgMaxBpSL = Integer.parseInt(prop.mustGet("LgMaxBpSL"));
+ //stream length
+ lgMinSL = Integer.parseInt(prop.mustGet("LgMinSL"));
+ lgMaxSL = Integer.parseInt(prop.mustGet("LgMaxSL"));
+ ppoSL = Integer.parseInt(prop.mustGet("PpoSL"));
+
+ //Target sketch config
+ kllK = Integer.parseInt(prop.mustGet("KllK"));
+ }
+
+ void configureCommon() {
+ slope = (double) (lgMaxT - lgMinT) / (lgMinBpSL - lgMaxBpSL);
+ }
+
+ void configureSketch() {
+ kllSk = new KllFloatsSketch(kllK);
+ }
+
+//JobProfile interface
+ @Override
+ public void start(final Job job) {
+ this.job = job;
+ prop = job.getProperties();
+ extractProperties();
+ configureCommon();
+ configureSketch();
+ doTrials();
+ }
+
+ @Override
+ public void shutdown() {}
+
+ @Override
+ public void cleanup() {}
+ //end JobProfile
+
+ /**
+ * Traverses all the axis plot points and performs trials(sl) at each point
+ * and outputs a row per axis plot point.
+ */
+ private void doTrials() {
+ final int maxSL = 1 << lgMaxSL;
+ final int minSL = 1 << lgMinSL;
+ int lastSL = 0;
+ job.printf(sFmt, (Object[]) columnLabels); //Header
+ int pp = 1;
+ while (lastSL < maxSL) { //Trials for each plotPoint on X-axis, and one row on output
+ final int nextSL = lastSL == 0 ? minSL : pwr2LawNext(ppoSL, lastSL);
+ lastSL = nextSL;
+ final int trials = getNumTrials(nextSL);
+
+ double sumUpdateTimePerItem_nS = 0;
+ for (int t = 0; t < trials; t++) {
+ sumUpdateTimePerItem_nS += doTrial(nextSL);
+ }
+ final double meanUpdateTimePerItem_nS = sumUpdateTimePerItem_nS / trials;
+ final int bytes = kllSk.getSerializedSizeBytes();
+ job.printf(dFmt, pp, nextSL, trials, bytes, meanUpdateTimePerItem_nS);
+ pp++;
+ }
+ }
+
+ /**
+ * Return the average update time per item for this trial
+ * @param streamLen the streamLength for this trial
+ * @return the average update time per item for this trial
+ */
+ private double doTrial(final int streamLen) {
+ kllSk = new KllFloatsSketch(kllK);
+ final long startUpdateTime_nS = System.nanoTime();
+
+ for (int i = 0; i < streamLen; i++) {
+ kllSk.update(i);
+ }
+ final long updateTime_nS = System.nanoTime() - startUpdateTime_nS;
+ return (double) updateTime_nS / streamLen;
+ }
+
+ /**
+ * Computes the number of trials for a given current stream length for a
+ * trial set. This is used in speed trials and decreases the number of trials
+ * as the stream length increases.
+ *
+ * @param curSL the given current stream length for a trial set.
+ * @return the number of trials for a given current stream length for a
+ * trial set.
+ */
+ private int getNumTrials(final int curSL) {
+ final int minBpSL = 1 << lgMinBpSL;
+ final int maxBpSL = 1 << lgMaxBpSL;
+ final int maxT = 1 << lgMaxT;
+ final int minT = 1 << lgMinT;
+ if (lgMinT == lgMaxT || curSL <= minBpSL) {
+ return maxT;
+ }
+ if (curSL >= maxBpSL) {
+ return minT;
+ }
+ final double lgCurU = log(curSL) / LN2;
+ final double lgTrials = slope * (lgCurU - lgMinBpSL) + lgMaxT;
+ return (int) pow(2.0, lgTrials);
+ }
+
+}
diff --git a/src/test/java/org/apache/datasketches/characterization/quantiles/ReqSketchAccuracyProfile.java b/src/test/java/org/apache/datasketches/characterization/quantiles/ReqSketchAccuracyProfile.java
index 57867d0..5ecbc98 100644
--- a/src/test/java/org/apache/datasketches/characterization/quantiles/ReqSketchAccuracyProfile.java
+++ b/src/test/java/org/apache/datasketches/characterization/quantiles/ReqSketchAccuracyProfile.java
@@ -20,21 +20,23 @@
package org.apache.datasketches.characterization.quantiles;
import static java.lang.Math.round;
+import static org.apache.datasketches.Criteria.GE;
+import static org.apache.datasketches.Criteria.LT;
import static org.apache.datasketches.ExponentiallySpacedPoints.expSpacedFloats;
import static org.apache.datasketches.GaussianRanks.GAUSSIANS_3SD;
import static org.apache.datasketches.Util.evenlySpacedFloats;
import static org.apache.datasketches.Util.pwr2LawNext;
-import static org.apache.datasketches.req.Criteria.GE;
-import static org.apache.datasketches.req.Criteria.LT;
+import org.apache.datasketches.Criteria;
import org.apache.datasketches.Job;
import org.apache.datasketches.JobProfile;
import org.apache.datasketches.MonotonicPoints;
+import org.apache.datasketches.Properties;
import org.apache.datasketches.characterization.Shuffle;
+import org.apache.datasketches.hll.HllSketch;
import org.apache.datasketches.quantiles.DoublesSketch;
import org.apache.datasketches.quantiles.DoublesSketchBuilder;
import org.apache.datasketches.quantiles.UpdateDoublesSketch;
-import org.apache.datasketches.req.Criteria;
import org.apache.datasketches.req.ReqDebugImpl;
import org.apache.datasketches.req.ReqSketch;
import org.apache.datasketches.req.ReqSketchBuilder;
@@ -44,6 +46,7 @@ import org.apache.datasketches.req.ReqSketchBuilder;
*/
public class ReqSketchAccuracyProfile implements JobProfile {
private Job job;
+ private Properties prop;
//FROM PROPERTIES
//For computing the different stream lengths
@@ -53,7 +56,8 @@ public class ReqSketchAccuracyProfile implements JobProfile {
private int ppo; //not currently used
private int numTrials; //num of Trials per plotPoint
- private int errorSkLgK; //size of the error quantiles sketches
+ private int errQSkLgK; //size of the error quantiles sketches
+ private int errHllSkLgK; //size of the error HLL sketch
//plotting & x-axis configuration
private int numPlotPoints;
@@ -65,6 +69,7 @@ public class ReqSketchAccuracyProfile implements JobProfile {
//Target sketch configuration & error analysis
private int K;
private boolean hra; //high rank accuracy
+ private boolean compatible;
private Criteria criterion;
private org.apache.datasketches.req.ReqDebugImpl reqDebugImpl = null;
@@ -74,26 +79,28 @@ public class ReqSketchAccuracyProfile implements JobProfile {
//The array of Gaussian quantiles for +/- StdDev error analysis
private double[] gRanks;
private UpdateDoublesSketch[] errQSkArr;
+ private HllSketch[] errHllSkArr;
//Specific to a streamLength
- private float[] stream;
- private float[] trueValues;
+ private float[] stream; //a shuffled array of values from 1...N
+ private float[] trueValues; //
private int trueValueCorrection;
private float[] corrTrueValues;
private final String[] columnLabels =
- {"rPP", "Value", "Rank", "-3SD","-2SD", "-1SD", "Med", "+1SD", "+2SD", "+3SD", "RLB", "RUB"};
+ {"nPP", "Value", "Rank", "-3SD","-2SD", "-1SD", "Med", "+1SD", "+2SD", "+3SD", "1LB", "1UB", "U"};
private final String sFmt =
- "%3s\t%5s\t%4s\t%4s\t%4s\t%4s\t%5s\t%4s\t%4s\t%4s\t%3s\t%3s\n";
+ "%3s\t%5s\t%4s\t%4s\t%4s\t%4s\t%5s\t%4s\t%4s\t%4s\t%3s\t%3s\t%3s\n";
private final String fFmt =
"%14.10f\t%14.0f\t%14.10f\t" //rPP, Value, Rank
+ "%14.10f\t%14.10f\t%14.10f\t%14.10f\t%14.10f\t%14.10f\t%14.10f\t" //-3sd to +3sd
- + "%14.10f\t%14.10f\n"; //rlb, rub
+ + "%14.10f\t%14.10f\t%6d\n"; //1lb, 1ub, U
//JobProfile interface
@Override
public void start(final Job job) {
this.job = job;
+ prop = job.getProperties();
extractProperties();
configureCommon();
doJob();
@@ -108,25 +115,27 @@ public class ReqSketchAccuracyProfile implements JobProfile {
private void extractProperties() {
//stream length
- lgMin = Integer.parseInt(job.getProperties().mustGet("LgMin"));
- lgMax = Integer.parseInt(job.getProperties().mustGet("LgMax"));
- lgDelta = Integer.parseInt(job.getProperties().mustGet("LgDelta"));
- ppo = Integer.parseInt(job.getProperties().mustGet("PPO"));
- //numTrials & error quantiles sketch config
- numTrials = 1 << Integer.parseInt(job.getProperties().mustGet("LgTrials"));
- errorSkLgK = Integer.parseInt(job.getProperties().mustGet("ErrSkLgK"));
+ lgMin = Integer.parseInt(prop.mustGet("LgMin"));
+ lgMax = Integer.parseInt(prop.mustGet("LgMax"));
+ lgDelta = Integer.parseInt(prop.mustGet("LgDelta"));
+ ppo = Integer.parseInt(prop.mustGet("PPO"));
+ //numTrials & error quantiles & HLL sketch config
+ numTrials = 1 << Integer.parseInt(prop.mustGet("LgTrials"));
+ errQSkLgK = Integer.parseInt(prop.mustGet("ErrQSkLgK"));
+ errHllSkLgK = Integer.parseInt(prop.mustGet("ErrHllSkLgK"));
//plotting & x-axis config
- numPlotPoints = Integer.parseInt(job.getProperties().mustGet("NumPlotPoints"));
- evenlySpaced = Boolean.valueOf(job.getProperties().mustGet("EvenlySpaced"));
- exponent = Double.parseDouble(job.getProperties().mustGet("Exponent"));
- sd = Integer.parseInt(job.getProperties().mustGet("StdDev"));
- rankRange = Double.parseDouble(job.getProperties().mustGet("RankRange"));
+ numPlotPoints = Integer.parseInt(prop.mustGet("NumPlotPoints"));
+ evenlySpaced = Boolean.valueOf(prop.mustGet("EvenlySpaced"));
+ exponent = Double.parseDouble(prop.mustGet("Exponent"));
+ sd = Integer.parseInt(prop.mustGet("StdDev"));
+ rankRange = Double.parseDouble(prop.mustGet("RankRange"));
//Target sketch config
- K = Integer.parseInt(job.getProperties().mustGet("K"));
- hra = Boolean.parseBoolean(job.getProperties().mustGet("HRA"));
- criterion = Criteria.valueOf(job.getProperties().mustGet("Criterion"));
- String reqDebugLevel = job.getProperties().get("ReqDebugLevel");
- String reqDebugFmt = job.getProperties().get("ReqDebugFmt");
+ K = Integer.parseInt(prop.mustGet("K"));
+ hra = Boolean.parseBoolean(prop.mustGet("HRA"));
+ compatible = Boolean.parseBoolean(prop.mustGet("Compatible"));
+ criterion = Criteria.valueOf(prop.mustGet("Criterion"));
+ String reqDebugLevel = prop.get("ReqDebugLevel");
+ String reqDebugFmt = prop.get("ReqDebugFmt");
if (reqDebugLevel != null) {
int level = Integer.parseInt(reqDebugLevel);
reqDebugImpl = new ReqDebugImpl(level, reqDebugFmt);
@@ -139,10 +148,12 @@ public class ReqSketchAccuracyProfile implements JobProfile {
corrTrueValues = new float[numPlotPoints];
trueValueCorrection = criterion == GE || criterion == LT ? 1 : 0;
errQSkArr = new UpdateDoublesSketch[numPlotPoints];
- //configure the error quantiles array
- final DoublesSketchBuilder builder = DoublesSketch.builder().setK(1 << errorSkLgK);
+ errHllSkArr = new HllSketch[numPlotPoints];
+ //configure the error quantiles array & HLL sketch arr
+ final DoublesSketchBuilder builder = DoublesSketch.builder().setK(1 << errQSkLgK);
for (int i = 0; i < numPlotPoints; i++) {
errQSkArr[i] = builder.build();
+ errHllSkArr[i] = new HllSketch(errHllSkLgK);
}
gRanks = new double[GAUSSIANS_3SD.length - 2]; //omit 0.0 and 1.0
for (int i = 1; i < GAUSSIANS_3SD.length - 1; i++) {
@@ -210,7 +221,7 @@ public class ReqSketchAccuracyProfile implements JobProfile {
//Do numTrials for all plotpoints
for (int t = 0; t < numTrials; t++) {
- doTrial(sk, stream, trueValues, corrTrueValues, errQSkArr);
+ doTrial(sk, stream, trueValues, corrTrueValues, errQSkArr, errHllSkArr);
}
//at this point each of the errQSkArr sketches has a distribution of error from numTrials
@@ -220,15 +231,18 @@ public class ReqSketchAccuracyProfile implements JobProfile {
final double rlb = sk.getRankLowerBound(tr, sd) - tr;
final double rub = sk.getRankUpperBound(tr, sd) - tr;
+
//for each of the numErrDistRanks distributions extract the sd quantiles
final double[] errQ = errQSkArr[pp].getQuantiles(gRanks); //get error values at the Gaussian ranks
+ final int errCnt = (int)round(errHllSkArr[pp].getEstimate());
- //Plot the row. We ignore quantiles collected at 0 and 1.0.
+ //Plot the row.
final double relPP = (double)(pp + 1) / numPlotPoints;
job.printfData(fFmt, relPP, v, tr,
errQ[0], errQ[1], errQ[2], errQ[3], errQ[4], errQ[5], errQ[6],
- rlb, rub);
+ rlb, rub, errCnt);
errQSkArr[pp].reset(); //reset the errQSkArr for next streamLength
+ errHllSkArr[pp].reset(); //reset the errHllSkArr for next streamLength
}
job.println(LS + "Serialization Bytes: " + sk.getSerializationBytes());
job.println(sk.viewCompactorDetail("%5.0f", false));
@@ -243,7 +257,7 @@ public class ReqSketchAccuracyProfile implements JobProfile {
* @param errQSkArr the quantile error sketches for each plot point to be updated
*/
static void doTrial(final ReqSketch sk, final float[] stream, final float[] trueValues,
- final float[] corrTrueValues, final UpdateDoublesSketch[] errQSkArr) {
+ final float[] corrTrueValues, final UpdateDoublesSketch[] errQSkArr, HllSketch[] errHllSkArr) {
sk.reset();
Shuffle.shuffle(stream);
final int sl = stream.length;
@@ -253,10 +267,11 @@ public class ReqSketchAccuracyProfile implements JobProfile {
//get estimated ranks from sketch for all plotpoints, this is a bulk operation
final double[] estRanks = sk.getRanks(trueValues);
final int numPP = trueValues.length;
- //compute errors for each plotPoint
+ //compute errors and update HLL for each plotPoint
for (int pp = 0; pp < numPP; pp++) {
final double errorAtPlotPoint = estRanks[pp] - (double)corrTrueValues[pp] / sl;
errQSkArr[pp].update(errorAtPlotPoint); //update each of the errQArr sketches
+ errHllSkArr[pp].update(errorAtPlotPoint);
}
}
diff --git a/src/test/java/org/apache/datasketches/characterization/quantiles/ReqSketchAccuracyProfile2.java b/src/test/java/org/apache/datasketches/characterization/quantiles/ReqSketchAccuracyProfile2.java
new file mode 100644
index 0000000..727eaf8
--- /dev/null
+++ b/src/test/java/org/apache/datasketches/characterization/quantiles/ReqSketchAccuracyProfile2.java
@@ -0,0 +1,245 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.datasketches.characterization.quantiles;
+
+import static org.apache.datasketches.GaussianRanks.GAUSSIANS_3SD;
+import static org.apache.datasketches.Util.evenlySpacedFloats;
+
+import org.apache.datasketches.Criteria;
+import org.apache.datasketches.Job;
+import org.apache.datasketches.JobProfile;
+import org.apache.datasketches.Properties;
+import org.apache.datasketches.characterization.quantiles.StreamMaker.Pattern;
+import org.apache.datasketches.quantiles.DoublesSketch;
+import org.apache.datasketches.quantiles.DoublesSketchBuilder;
+import org.apache.datasketches.quantiles.UpdateDoublesSketch;
+import org.apache.datasketches.req.ReqDebugImpl;
+import org.apache.datasketches.req.ReqSketch;
+import org.apache.datasketches.req.ReqSketchBuilder;
+
+/**
+ * @author Lee Rhodes
+ */
+public class ReqSketchAccuracyProfile2 implements JobProfile {
+ private Job job;
+ private Properties prop;
+ Pattern pattern;
+
+ //PROPERTIES
+ //plotting & x-axis configuration
+ private int lgSL;
+ private int numPlotPoints;
+ private int stdDev;
+
+ //Patterns
+ private int offset; //Stream offset, 0 or 1
+// private int[] advSeq1 = new int[3];
+// private int[] advSeq2 = new int[3];
+// private int[] advSeq3 = new int[3];
+
+ private int errQSkLgK;
+
+ //TargetSketch config & error analysis
+ private int K;
+ private boolean hra;
+ private boolean compatible;
+ private Criteria criterion;
+ private org.apache.datasketches.req.ReqDebugImpl reqDebugImpl = null;
+
+ //DERIVED INTERNAL globals
+ private ReqSketch sk;
+ private int N;
+
+ //The array of Gaussian quantiles for +/- StdDev error analysis
+ private double[] gRanks;
+ private UpdateDoublesSketch[] errQSkArr;
+
+ //Specific to the stream
+ private TrueRanks trueRanks;
+ private float[] sortedPPValues;
+ private int[] sortedPPIndices;
+ private int[] sortedPPAbsRanks;
+
+ private final String[] columnLabels =
+ { "PP", "Value", "TrueRanks",
+ "-3SD","-2SD", "-1SD", "Med", "+1SD", "+2SD", "+3SD",
+ "1LB", "1UB" };
+ private final String sFmt =
+ "%2s\t%5s\t%9s\t"
+ + "%4s\t%4s\t%4s\t%3s\t%4s\t%4s\t%4s\t"
+ + "%3s\t%3s\n";
+ private final String fFmt =
+ "%5d\t%,14.0f\t%14.10f\t" //nPP, Value, Rank
+ + "%14.10f\t%14.10f\t%14.10f\t%14.10f\t%14.10f\t%14.10f\t%14.10f\t" //-3sd to +3sd
+ + "%14.10f\t%14.10f\n"; //1lb, 1ub
+
+ //JobProfile interface
+ @Override
+ public void start(Job job) {
+ this.job = job;
+ prop = job.getProperties();
+ extractProperties();
+ configureSketch();
+ configureStream();
+ configurePlotPoints();
+ doStreamLength(N);
+ }
+
+ @Override
+ public void shutdown() { }
+
+ @Override
+ public void cleanup() { }
+ //end JobProfile
+
+ private void extractProperties() {
+ //plotting & x-axis configuration
+ lgSL = Integer.parseInt(prop.mustGet("LgSL"));
+ numPlotPoints = Integer.parseInt(prop.mustGet("NumPlotPoints"));
+ stdDev = Integer.parseInt(prop.mustGet("StdDev"));
+ //Patterns
+ pattern = Pattern.valueOf(prop.mustGet("Pattern"));
+ offset = Integer.parseInt(prop.mustGet("Offset"));
+// String[] sarr = prop.mustGet("AdvSeq1").split(",", 3);
+// for (int i = 0; i < sarr.length; i++) { advSeq1[i] = Integer.parseInt(sarr[i]); }
+// sarr = prop.mustGet("AdvSeq2").split(",", 3);
+// for (int i = 0; i < sarr.length; i++) { advSeq2[i] = Integer.parseInt(sarr[i]); }
+// sarr = prop.mustGet("AdvSeq3").split(",", 3);
+// for (int i = 0; i < sarr.length; i++) { advSeq3[i] = Integer.parseInt(sarr[i]); }
+
+ // error quantiles & HLL sketch config
+ errQSkLgK = Integer.parseInt(prop.mustGet("ErrQSkLgK"));
+ //plotting & x-axis config
+
+ //Target sketch config
+ K = Integer.parseInt(prop.mustGet("K"));
+ hra = Boolean.parseBoolean(prop.mustGet("HRA"));
+ criterion = Criteria.valueOf(prop.mustGet("Criterion"));
+ compatible = Boolean.parseBoolean(prop.mustGet("Compatible"));
+ String reqDebugLevel = prop.get("ReqDebugLevel");
+ String reqDebugFmt = prop.get("ReqDebugFmt");
+ if (reqDebugLevel != null) {
+ int level = Integer.parseInt(reqDebugLevel);
+ reqDebugImpl = new ReqDebugImpl(level, reqDebugFmt);
+ }
+ }
+
+ private void configureSketch() {
+ final ReqSketchBuilder bldr = ReqSketch.builder();
+ bldr.setK(K).setHighRankAccuracy(hra);
+ bldr.setCompatible(compatible);
+ if (reqDebugImpl != null) { bldr.setReqDebug(reqDebugImpl); }
+ sk = bldr.build();
+ sk.setCriterion(criterion);
+ }
+
+ private void configureStream() {
+ N = 1 << lgSL;
+ StreamMaker sm = new StreamMaker();
+ float[] stream = sm.makeStream(N, pattern, offset);
+ if (criterion == Criteria.LT) {
+ trueRanks = new TrueRanks(stream, false);
+ } else {
+ trueRanks = new TrueRanks(stream, true);
+ }
+ }
+
+ private void configurePlotPoints() {
+ sortedPPIndices = new int[numPlotPoints];
+ sortedPPAbsRanks = new int[numPlotPoints];
+ sortedPPValues = new float[numPlotPoints];
+ final int[] sortedAbsRanks = trueRanks.getSortedAbsRanks();
+ final float[] sortedStream = trueRanks.getSortedStream();
+ int minIdx = (int)Math.round((double)(N - 1) / numPlotPoints);
+ final float[] temp = evenlySpacedFloats(minIdx, N - 1, numPlotPoints);
+ for (int pp = 0; pp < numPlotPoints; pp++) {
+ final int idx = Math.round(temp[pp]);
+ sortedPPIndices[pp] = idx;
+ sortedPPAbsRanks[pp] = sortedAbsRanks[idx];
+ sortedPPValues[pp] = sortedStream[idx];
+ }
+
+ //configure the error quantiles array
+ errQSkArr = new UpdateDoublesSketch[numPlotPoints];
+ final DoublesSketchBuilder builder = DoublesSketch.builder().setK(1 << errQSkLgK);
+ for (int pp = 0; pp < numPlotPoints; pp++) {
+ errQSkArr[pp] = builder.build();
+ }
+ gRanks = new double[GAUSSIANS_3SD.length - 2]; //omit 0.0 and 1.0
+ for (int i = 1; i < GAUSSIANS_3SD.length - 1; i++) {
+ gRanks[i - 1] = GAUSSIANS_3SD[i];
+ }
+ }
+
+ private void doStreamLength(int streamLength) {
+ job.println(LS + "Stream Length: " + streamLength );
+ job.printfData(sFmt, (Object[])columnLabels);
+
+ doTrial();
+
+ //at this point each of the errQSkArr sketches has a distribution of error
+ for (int pp = 0 ; pp < numPlotPoints; pp++) {
+ final double tr = (double)sortedPPAbsRanks[pp] / N;
+ final float v = sortedPPValues[pp];
+ final double rlb = sk.getRankLowerBound(tr, stdDev) - tr;
+ final double rub = sk.getRankUpperBound(tr, stdDev) - tr;
+
+ //for each of the numErrDistRanks distributions extract the sd quantiles
+ final double[] errQ = errQSkArr[pp].getQuantiles(gRanks); //get error values at the Gaussian ranks
+ if (errQ != null) {
+ //Plot the row.
+ job.printfData(fFmt, pp + 1, v, tr,
+ errQ[0], errQ[1], errQ[2], errQ[3], errQ[4], errQ[5], errQ[6],
+ rlb, rub);
+ }
+ errQSkArr[pp].reset(); //reset the errQSkArr for next streamLength
+ }
+ job.println(LS + "Serialization Bytes: " + sk.getSerializationBytes());
+ job.println(sk.viewCompactorDetail("%5.0f", false));
+ }
+
+ void doTrial() { //for all plot points
+ sk.reset();
+ final float[] stream = trueRanks.getStream();
+ for (int i = 0; i < N; i++) { sk.update(stream[i]); }
+
+ final float[] sortedStream = trueRanks.getSortedStream();
+ final int[] sortedAbsRanks = trueRanks.getSortedAbsRanks();
+
+ int pp = 0;
+ int ppAbsIdx = sortedPPIndices[pp];
+ UpdateDoublesSketch qSk = errQSkArr[pp];
+ for (int idx = 0; idx < N; idx++) {
+ final double skRank = sk.getRank(sortedStream[idx]);
+ double trueRank = (double)sortedAbsRanks[idx] / N;
+ if (idx <= ppAbsIdx) {
+ final double rErr = skRank - trueRank;
+ qSk.update(rErr);
+ } else {
+ if (++pp < numPlotPoints) {
+ ppAbsIdx = sortedPPIndices[pp];
+ qSk = errQSkArr[pp];
+ final double rErr = skRank - trueRank;
+ qSk.update(rErr);
+ } else { break; }
+ }
+ }
+ }
+}
diff --git a/src/test/java/org/apache/datasketches/characterization/quantiles/ReqSketchSizeSpeedProfile.java b/src/test/java/org/apache/datasketches/characterization/quantiles/ReqSketchSizeSpeedProfile.java
new file mode 100644
index 0000000..8fe87dd
--- /dev/null
+++ b/src/test/java/org/apache/datasketches/characterization/quantiles/ReqSketchSizeSpeedProfile.java
@@ -0,0 +1,178 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.datasketches.characterization.quantiles;
+
+import static java.lang.Math.log;
+import static java.lang.Math.pow;
+import static org.apache.datasketches.Util.pwr2LawNext;
+
+import org.apache.datasketches.Criteria;
+import org.apache.datasketches.Job;
+import org.apache.datasketches.JobProfile;
+import org.apache.datasketches.Properties;
+import org.apache.datasketches.req.ReqSketch;
+import org.apache.datasketches.req.ReqSketchBuilder;
+
+/**
+ * @author Lee Rhodes
+ */
+public class ReqSketchSizeSpeedProfile implements JobProfile {
+ private Job job;
+ private Properties prop;
+
+ //FROM PROPERTIES
+ private int lgMinT;
+ private int lgMaxT;
+ private int lgMinBpSL;
+ private int lgMaxBpSL;
+ //For computing the different stream lengths
+ private int lgMinSL;
+ private int lgMaxSL;
+ private int ppoSL;
+
+ private double slope;
+
+ //Target sketch configuration & error analysis
+ private int reqK;
+ private boolean hra; //high rank accuracy
+ private Criteria criterion;
+
+ //DERIVED & GLOBALS
+ private ReqSketch reqSk;
+ //private KllFloatsSketch kllSk;
+
+ private final String[] columnLabels = {"PP", "SL", "Trials", "ReqBytes", "nS/u" };
+ private final String sFmt = "%2s\t%2s\t%6s\t%8s\t%4s\n";
+ private final String dFmt = "%,6d\t%,12d\t%,12d\t%,12d\t%,12.6f\n";
+
+ private void extractProperties() {
+ //trials config
+ lgMinT = Integer.parseInt(prop.mustGet("LgMinT"));
+ lgMaxT = Integer.parseInt(prop.mustGet("LgMaxT"));
+ lgMinBpSL = Integer.parseInt(prop.mustGet("LgMinBpSL"));
+ lgMaxBpSL = Integer.parseInt(prop.mustGet("LgMaxBpSL"));
+ //stream length
+ lgMinSL = Integer.parseInt(prop.mustGet("LgMinSL"));
+ lgMaxSL = Integer.parseInt(prop.mustGet("LgMaxSL"));
+ ppoSL = Integer.parseInt(prop.mustGet("PpoSL"));
+
+ //Target sketch config
+ reqK = Integer.parseInt(prop.mustGet("ReqK"));
+ hra = Boolean.parseBoolean(prop.mustGet("HRA"));
+ criterion = Criteria.valueOf(prop.mustGet("Criterion"));
+ }
+
+ void configureCommon() {
+ slope = (double) (lgMaxT - lgMinT) / (lgMinBpSL - lgMaxBpSL);
+ }
+
+ void configureSketch() {
+ final ReqSketchBuilder bldr = ReqSketch.builder();
+ bldr.setK(reqK).setHighRankAccuracy(hra);
+ reqSk = bldr.build();
+ reqSk.setCriterion(criterion);
+ }
+
+//JobProfile interface
+ @Override
+ public void start(final Job job) {
+ this.job = job;
+ prop = job.getProperties();
+ extractProperties();
+ configureCommon();
+ configureSketch();
+ doTrials();
+ }
+
+ @Override
+ public void shutdown() {}
+
+ @Override
+ public void cleanup() {}
+ //end JobProfile
+
+ /**
+ * Traverses all the axis plot points and performs trials(sl) at each point
+ * and outputs a row per axis plot point.
+ */
+ private void doTrials() {
+ final int maxSL = 1 << lgMaxSL;
+ final int minSL = 1 << lgMinSL;
+ int lastSL = 0;
+ job.printf(sFmt, (Object[]) columnLabels); //Header
+ int pp = 1;
+ while (lastSL < maxSL) { //Trials for each plotPoint on X-axis, and one row on output
+ final int nextSL = lastSL == 0 ? minSL : pwr2LawNext(ppoSL, lastSL);
+ lastSL = nextSL;
+ final int trials = getNumTrials(nextSL);
+
+ double sumUpdateTimePerItem_nS = 0;
+ for (int t = 0; t < trials; t++) {
+ sumUpdateTimePerItem_nS += doTrial(nextSL);
+ }
+ final double meanUpdateTimePerItem_nS = sumUpdateTimePerItem_nS / trials;
+ final int bytes = reqSk.getSerializationBytes();
+ job.printf(dFmt, pp, nextSL, trials, bytes, meanUpdateTimePerItem_nS);
+ pp++;
+ }
+ }
+
+ /**
+ * Return the average update time per item for this trial
+ * @param streamLen the streamLength for this trial
+ * @return the average update time per item for this trial
+ */
+ private double doTrial(final int streamLen) {
+ reqSk.reset();
+ final long startUpdateTime_nS = System.nanoTime();
+
+ for (int i = 0; i < streamLen; i++) {
+ reqSk.update(i);
+ }
+ final long updateTime_nS = System.nanoTime() - startUpdateTime_nS;
+ return (double) updateTime_nS / streamLen;
+ }
+
+ /**
+ * Computes the number of trials for a given current stream length for a
+ * trial set. This is used in speed trials and decreases the number of trials
+ * as the stream length increases.
+ *
+ * @param curSL the given current stream length for a trial set.
+ * @return the number of trials for a given current stream length for a
+ * trial set.
+ */
+ private int getNumTrials(final int curSL) {
+ final int minBpSL = 1 << lgMinBpSL;
+ final int maxBpSL = 1 << lgMaxBpSL;
+ final int maxT = 1 << lgMaxT;
+ final int minT = 1 << lgMinT;
+ if (lgMinT == lgMaxT || curSL <= minBpSL) {
+ return maxT;
+ }
+ if (curSL >= maxBpSL) {
+ return minT;
+ }
+ final double lgCurU = log(curSL) / LN2;
+ final double lgTrials = slope * (lgCurU - lgMinBpSL) + lgMaxT;
+ return (int) pow(2.0, lgTrials);
+ }
+
+}
diff --git a/src/test/java/org/apache/datasketches/characterization/quantiles/StreamMaker.java b/src/test/java/org/apache/datasketches/characterization/quantiles/StreamMaker.java
new file mode 100644
index 0000000..2c2cf77
--- /dev/null
+++ b/src/test/java/org/apache/datasketches/characterization/quantiles/StreamMaker.java
@@ -0,0 +1,146 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.datasketches.characterization.quantiles;
+
+import static org.apache.datasketches.characterization.quantiles.StreamMaker.Pattern.FlipFlop;
+import static org.apache.datasketches.characterization.quantiles.StreamMaker.Pattern.Random;
+import static org.apache.datasketches.characterization.quantiles.StreamMaker.Pattern.Reversed;
+import static org.apache.datasketches.characterization.quantiles.StreamMaker.Pattern.Sorted;
+import static org.apache.datasketches.characterization.quantiles.StreamMaker.Pattern.Sqrt;
+import static org.apache.datasketches.characterization.quantiles.StreamMaker.Pattern.Zoomin;
+import static org.apache.datasketches.characterization.quantiles.StreamMaker.Pattern.Zoomout;
+
+import org.apache.datasketches.characterization.Shuffle;
+import org.testng.annotations.Test;
+
+/**
+ * @author Lee Rhodes
+ */
+public class StreamMaker {
+ public enum Pattern { Sorted, Reversed, Zoomin, Zoomout, Random, Sqrt, FlipFlop,
+ Clustered, ClusteredZoomin }
+ public float min = 0;
+ public float max = 0;
+
+ public float[] makeStream(int n, Pattern pattern, int offset) {
+ float[] arr = new float[n];
+ min = offset;
+ max = n - 1 + offset;
+ switch(pattern) {
+ case Sorted: {
+ for (int i = 0; i < n; i++) { arr[i] = i + offset; }
+ break;
+ }
+ case Reversed: {
+ for (int i = 0; i < n; i++) { arr[n - 1 - i] = i + offset; }
+ break;
+ }
+ case Zoomin: {
+ for (int i = 0, j = 0; i < n; i++) {
+ if ((i & 1) > 0) { arr[i] = n - j - 1 + offset; j++; } //odd
+ else { arr[i] = j + offset; }
+ }
+ break;
+ }
+ case Zoomout: {
+ for (int i = 0, j = 0; i < n; i++) {
+ if ((i & 1) > 0) { arr[n - 1 - i] = n - j - 1 + offset; j++; } //odd
+ else { arr[n - 1 - i] = j + offset; }
+ }
+ break;
+ }
+ case Random: {
+ for (int i = 0; i < n; i++) { arr[i] = i + offset; }
+ Shuffle.shuffle(arr);
+ break;
+ }
+ case Sqrt: {
+ int idx = 0;
+ int t = (int)Math.sqrt(2 * n);
+ int item = 0;
+ int initialItem = 0;
+ int initialSkip = 1;
+ for (int i = 0; i < t; i++) {
+ item = initialItem;
+ int skip = initialSkip;
+ for (int j = 0; j < t - i; j++) {
+ if (idx > n - 1) { break; }
+ arr[idx++] = item + offset;
+ item += skip;
+ skip += 1;
+ }
+ if (idx > n - 1) { break; }
+ initialSkip += 1;
+ initialItem += initialSkip;
+ }
+ break;
+ }
+ case FlipFlop: {
+ FlipFlopStream ffs = new FlipFlopStream(n, offset);
+ ffs.flipFlop(1, 1, n * 2 / 5);
+ int m = n / 5;
+ ffs.flipFlop(m, 1, m);
+ ffs.flipFlop(1, m, m);
+ ffs.flipFlop(1, 1, n);
+ arr = ffs.getArray();
+ break;
+ }
+ case Clustered: {
+ break;
+ }
+ case ClusteredZoomin: {
+ break;
+ }
+ }
+ return arr;
+ }
+
+ public void printStream(int n, Pattern order, int offset) {
+ float[] stream = makeStream(n, order, offset);
+ println(order + " n:" + n + " offset: " + offset);
+ for (int i = 0; i < stream.length; i++) {
+ if (i != 0 && i % 21 == 0) { println(""); }
+ print(stream[i] + " ");
+ }
+ println("");
+ }
+
+ @Test
+ public void checkStreamMaker() {
+ printStream(20, Sorted, 0);
+ printStream(20, Reversed, 0);
+ printStream(20, Zoomin, 0);
+ printStream(21, Zoomin, 1);
+ printStream(20, Zoomout, 0);
+ printStream(21, Zoomout, 1);
+ printStream(21, Random, 0);
+ printStream(21, Random, 1);
+ printStream(20, Sqrt, 0);
+ printStream(20, Sqrt, 1);
+ printStream(21, Sqrt, 0);
+ printStream(21, Sqrt, 1);
+ printStream(71, FlipFlop, 0);
+ printStream(71, FlipFlop, 1);
+ }
+
+ static void print(Object o) { System.out.print(o.toString()); }
+
+ static void println(Object o) { System.out.println(o.toString()); }
+}
diff --git a/src/test/java/org/apache/datasketches/characterization/quantiles/TrueRanks.java b/src/test/java/org/apache/datasketches/characterization/quantiles/TrueRanks.java
new file mode 100644
index 0000000..f725f22
--- /dev/null
+++ b/src/test/java/org/apache/datasketches/characterization/quantiles/TrueRanks.java
@@ -0,0 +1,172 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.datasketches.characterization.quantiles;
+
+import java.util.Arrays;
+
+import org.apache.datasketches.BinarySearch;
+import org.apache.datasketches.characterization.Shuffle;
+import org.testng.annotations.Test;
+
+/**
+ * Given an array of values, these methods compute the true rank (mass) of
+ * each value of the array based on the comparison criterion.
+ * The mass or rank of each value is the fractional number of elements of the array that satisfy
+ * the criterion.
+ *
+ * @author Lee Rhodes
+ */
+public class TrueRanks {
+ private static final String LS = System.getProperty("line.separator");
+ private boolean ltEq;
+ private int length;
+ private float[] stream;
+ private float[] sortedStream;
+ private int[] sortedAbsRanks;
+ private int[] streamAbsRanks; //??
+
+ TrueRanks() { } //for TestNG
+
+ public TrueRanks(final float[] stream, final boolean ltEq) {
+ this.stream = stream;
+ this.ltEq = ltEq;
+ compute();
+ }
+
+ public float getMinValue() { return sortedStream[0]; }
+
+ public float getMaxValue() { return sortedStream[length - 1]; }
+
+ public int getMinAbsRank() { return sortedAbsRanks[0]; }
+
+ public int getMaxAbsRank() { return sortedAbsRanks[length - 1]; }
+
+ public float[] getStream() { return stream; }
+
+ public float[] getSortedStream() { return sortedStream; }
+
+ public int[] getSortedAbsRanks() { return sortedAbsRanks; }
+
+ public int[] getStreamAbsRanks() { return streamAbsRanks; }
+
+ public double[] getSortedRelRanks() {
+ return relativeRank(sortedAbsRanks);
+ }
+
+ public double[] getStreamRelRanks() {
+ return relativeRank(streamAbsRanks);
+ }
+
+ public int getAbsRank(final float v) {
+ int idx = BinarySearch.find(sortedStream, 0, length - 1, v);
+ return sortedAbsRanks[idx];
+ }
+
+ /**
+ * Compute Absolute Rank.
+ */
+ private void compute() {
+ length = stream.length;
+ sortedStream = stream.clone();
+ Arrays.sort(sortedStream);
+ sortedAbsRanks = new int[length];
+ if (ltEq) { //LE
+ sortedAbsRanks[length - 1] = length;
+ int i = length - 2;
+ while (i >= 0) {
+ if (sortedStream[i] == sortedStream[i + 1]) { sortedAbsRanks[i] = sortedAbsRanks[i + 1]; }
+ else { sortedAbsRanks[i] = i + 1; }
+ i--;
+ }
+ } else { // LT
+ sortedAbsRanks[0] = 0;
+ int i = 1;
+ while (i < length) {
+ if (sortedStream[i - 1] == sortedStream[i]) { sortedAbsRanks[i] = sortedAbsRanks[i - 1]; }
+ else { sortedAbsRanks[i] = i; }
+ i++;
+ }
+ }
+ streamAbsRanks = new int[length]; //put the ranks in values order
+ for (int j = 0; j < length; j++) {
+ final int idx = BinarySearch.find(sortedStream, 0, length - 1, stream[j]);
+ streamAbsRanks[j] = sortedAbsRanks[idx];
+ }
+ }
+
+ /**
+ * Converts an absolute rank array to a relative rank array.
+ * @param absRankArr the absolute rank array to be converted.
+ * @return the relative rank array.
+ */
+ public static double[] relativeRank(final int[] absRankArr) {
+ int length = absRankArr.length;
+ double[] relRank = new double[length];
+ for (int i = 0; i < length; i++) { relRank[i] = (double)absRankArr[i] / length; }
+ return relRank;
+ }
+
+ @Test
+ public void checkRanks() {
+ final float[] vArr = { 5, 5, 5, 6, 6, 6, 7, 8, 8, 8 };
+ checkRanksImpl(vArr);
+ println(LS + "SHUFFLED:");
+ Shuffle.shuffle(vArr);
+ checkRanksImpl(vArr);
+ }
+
+ private static void checkRanksImpl(final float[] vArr) {
+ StringBuilder sb = new StringBuilder();
+ String ffmt = "%5.1f ";
+ String dfmt = "%5d ";
+ TrueRanks trueRanks;
+
+ int N = vArr.length;
+ sb.append("Values:").append(LS);
+ for (int i = 0; i < N; i++) { sb.append(String.format(ffmt, vArr[i])); }
+ sb.append(LS);
+
+ trueRanks = new TrueRanks(vArr, false);
+ sb.append("LT Abs Ranks:").append(LS);
+ int[] absArr = trueRanks.getStreamAbsRanks();
+ for (int i = 0; i < N; i++) { sb.append(String.format(dfmt, absArr[i])); }
+ sb.append(LS);
+ sb.append("LT Rel Ranks:").append(LS);
+ double[] relArr = relativeRank(absArr);
+ for (int i = 0; i < N; i++) { sb.append(String.format(ffmt, relArr[i])); }
+ sb.append(LS);
+
+ trueRanks = new TrueRanks(vArr, true);
+ sb.append("LE Abs Ranks:").append(LS);
+ absArr = trueRanks.getStreamAbsRanks();
+ for (int i = 0; i < N; i++) { sb.append(String.format(dfmt, absArr[i])); }
+ sb.append(LS);
+ sb.append("LE Rel Ranks:").append(LS);
+ relArr = relativeRank(absArr);
+ for (int i = 0; i < N; i++) { sb.append(String.format(ffmt, relArr[i])); }
+ sb.append(LS);
+ println(sb.toString());
+ }
+
+ private static void println(Object o) {
+ System.out.println(o.toString());
+ }
+
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org
For additional commands, e-mail: commits-help@datasketches.apache.org