You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2017/09/23 10:17:32 UTC
[01/13] kylin git commit: Fix smoke test result data error [Forced
Update!]
Repository: kylin
Updated Branches:
refs/heads/pr70 bd54abea2 -> f0bc691ec (forced update)
Fix smoke test result data error
Signed-off-by: lidongsjtu <li...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ff14042f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ff14042f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ff14042f
Branch: refs/heads/pr70
Commit: ff14042f5f318b1cc7881b06a4613c1de4439d2d
Parents: af8d885
Author: nichunen <ch...@kyligence.io>
Authored: Thu Sep 21 12:00:56 2017 +0800
Committer: lidongsjtu <li...@apache.org>
Committed: Thu Sep 21 13:28:25 2017 +0800
----------------------------------------------------------------------
build/smoke-test/testQuery.py | 21 +++++++++++----------
1 file changed, 11 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/ff14042f/build/smoke-test/testQuery.py
----------------------------------------------------------------------
diff --git a/build/smoke-test/testQuery.py b/build/smoke-test/testQuery.py
index 6d43e83..f0eec07 100644
--- a/build/smoke-test/testQuery.py
+++ b/build/smoke-test/testQuery.py
@@ -21,6 +21,7 @@ import unittest
import requests
import json
import glob
+import time
class testQuery(unittest.TestCase):
@@ -101,18 +102,18 @@ class testQuery(unittest.TestCase):
actual_result = json.loads(response.text)
print actual_result
- print 'Query duration: ' + str(actual_result['data']['duration']) + 'ms'
- del actual_result['data']['duration']
- del actual_result['data']['hitExceptionCache']
- del actual_result['data']['storageCacheUsed']
- del actual_result['data']['totalScanCount']
- del actual_result['data']['totalScanBytes']
- del actual_result['data']['columnMetas']
+ print 'Query duration: ' + str(actual_result['duration']) + 'ms'
+ del actual_result['duration']
+ del actual_result['hitExceptionCache']
+ del actual_result['storageCacheUsed']
+ del actual_result['totalScanCount']
+ del actual_result['totalScanBytes']
+ del actual_result['columnMetas']
expect_result = json.loads(open(sql_file[:-4] + '.json').read().strip())
- del expect_result['data']['columnMetas']
- expect_result['data']['cube'] = ''
- expect_result['data']['pushDown'] = True
+ del expect_result['columnMetas']
+ expect_result['cube'] = ''
+ expect_result['pushDown'] = True
self.assertEqual(actual_result, expect_result, 'Query pushdown\'s result does not equal with expected result.')
[03/13] kylin git commit: KYLIN-2870 Shortcut key description is
error at Kylin-Web
Posted by li...@apache.org.
KYLIN-2870 Shortcut key description is error at Kylin-Web
Signed-off-by: Li Yang <li...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5c1a3d10
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5c1a3d10
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5c1a3d10
Branch: refs/heads/pr70
Commit: 5c1a3d10fcdc15c2dc17dd3eae9668524195748b
Parents: ae430ed
Author: peng.jianhua <pe...@zte.com.cn>
Authored: Thu Sep 21 17:27:57 2017 +0800
Committer: Li Yang <li...@apache.org>
Committed: Sat Sep 23 11:53:03 2017 +0800
----------------------------------------------------------------------
webapp/app/partials/query/query.html | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/5c1a3d10/webapp/app/partials/query/query.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/query/query.html b/webapp/app/partials/query/query.html
index 894113e..5117bd8 100644
--- a/webapp/app/partials/query/query.html
+++ b/webapp/app/partials/query/query.html
@@ -59,7 +59,7 @@
</section>
<p class="blue">
- Tips: Ctrl+Shift+Space or Alt+Space(Windows), Command+Option+Space(Mac) to list tables/columns in query box.
+ Tips: Ctrl+Shift+Space or Alt+Space(Windows), Command+Option+Space(Mac) to list keywords in query box.
</p>
<div class="pull-left">
<h4>
[07/13] kylin git commit: KYLIN-2622 fix tests
Posted by li...@apache.org.
KYLIN-2622 fix tests
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2551d489
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2551d489
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2551d489
Branch: refs/heads/pr70
Commit: 2551d489ba774a1d6b67fd52b29b372557ab05ee
Parents: 7530edb
Author: Li Yang <li...@apache.org>
Authored: Sat Sep 23 16:42:49 2017 +0800
Committer: Li Yang <li...@apache.org>
Committed: Sat Sep 23 16:42:49 2017 +0800
----------------------------------------------------------------------
.../dict/global/SegmentAppendTrieDictBuilder.java | 15 ++++++---------
1 file changed, 6 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/2551d489/core-dictionary/src/main/java/org/apache/kylin/dict/global/SegmentAppendTrieDictBuilder.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/global/SegmentAppendTrieDictBuilder.java b/core-dictionary/src/main/java/org/apache/kylin/dict/global/SegmentAppendTrieDictBuilder.java
index 270deee..c8bc13d 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/global/SegmentAppendTrieDictBuilder.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/global/SegmentAppendTrieDictBuilder.java
@@ -18,14 +18,14 @@
package org.apache.kylin.dict.global;
+import java.io.IOException;
+import java.util.UUID;
+
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.Dictionary;
import org.apache.kylin.dict.DictionaryInfo;
import org.apache.kylin.dict.IDictionaryBuilder;
-import java.io.IOException;
-import java.util.UUID;
-
/**
* SegmentAppendTrieDictBuilder based on one segment.
* SegmentAppendTrieDictBuilder only used for count distinct measure that needn't rollup among segments.
@@ -41,14 +41,11 @@ public class SegmentAppendTrieDictBuilder implements IDictionaryBuilder {
public void init(DictionaryInfo dictInfo, int baseId) throws IOException {
sourceColumn = dictInfo.getSourceTable() + "." + dictInfo.getSourceColumn();
- int maxEntriesPerSlice = KylinConfig.getInstanceFromEnv().getAppendDictEntrySize();
- if (hdfsDir == null) {
- //build in Kylin job server
- hdfsDir = KylinConfig.getInstanceFromEnv().getHdfsWorkingDirectory();
- }
+ KylinConfig config = KylinConfig.getInstanceFromEnv();
+ int maxEntriesPerSlice = config.getAppendDictEntrySize();
//use UUID to make each segment dict in different HDFS dir and support concurrent build
//use timestamp to make the segment dict easily to delete
- String baseDir = KylinConfig.getInstanceFromEnv().getHdfsWorkingDirectory() + "resources/SegmentDict" + dictInfo.getResourceDir() + "/" + UUID.randomUUID().toString() + "_" + System.currentTimeMillis()+ "/";
+ String baseDir = config.getHdfsWorkingDirectory() + "resources/SegmentDict" + dictInfo.getResourceDir() + "/" + UUID.randomUUID().toString() + "_" + System.currentTimeMillis()+ "/";
this.builder = new AppendTrieDictionaryBuilder(baseDir, maxEntriesPerSlice, false);
this.baseId = baseId;
[12/13] kylin git commit: KYLIN-2826 add basic support classes for
cube planner algorithms
Posted by li...@apache.org.
http://git-wip-us.apache.org/repos/asf/kylin/blob/e2029dd9/core-cube/src/test/resources/statistics.txt
----------------------------------------------------------------------
diff --git a/core-cube/src/test/resources/statistics.txt b/core-cube/src/test/resources/statistics.txt
new file mode 100644
index 0000000..35f69fc
--- /dev/null
+++ b/core-cube/src/test/resources/statistics.txt
@@ -0,0 +1,4092 @@
+1 5408
+2 12
+3 59114
+4 5393
+5 13545958
+6 59352
+7 48900622
+8 234
+9 622632
+10 2513
+11 2684805
+12 661703
+13 51457071
+14 3003220
+15 87905334
+16 4
+17 20057
+18 48
+19 185091
+20 21381
+21 27129912
+22 196464
+23 67684891
+24 905
+25 1409380
+26 8032
+27 5367438
+28 1540851
+29 66914540
+30 5780199
+31 102382118
+32 5
+33 26824
+34 60
+35 250433
+36 26763
+37 30256052
+38 261493
+39 69394709
+40 387
+41 665893
+42 2849
+43 2954813
+44 707431
+45 61390587
+46 3240484
+47 97487859
+48 16
+49 78962
+50 189
+51 648097
+52 80270
+53 46703642
+54 675226
+55 85512868
+56 1138
+57 1463428
+58 8619
+59 5848317
+60 1577917
+61 77312551
+62 6472745
+63 111122050
+64 14
+65 42855
+66 144
+67 337000
+68 44890
+69 33660047
+70 361045
+71 73187752
+72 427
+73 715905
+74 3192
+75 3277233
+76 765421
+77 66688961
+78 3649848
+79 100898382
+80 40
+81 116673
+82 404
+83 846531
+84 123061
+85 51207630
+86 908398
+87 89543966
+89 1645138
+90 9255
+91 6589067
+92 1780584
+93 83045129
+94 7349741
+95 113014051
+96 54
+97 170610
+98 585
+99 1112616
+100 182519
+101 46860213
+102 1270670
+103 83382182
+104 596
+105 823678
+106 3734
+107 3900149
+108 884212
+109 74220549
+110 4350663
+111 106240989
+112 149
+113 407851
+114 1495
+115 2349130
+116 436830
+117 63626889
+118 2661185
+119 98988611
+120 1491
+121 1871944
+122 10494
+123 7619644
+124 2048333
+125 89765741
+126 8703141
+127 117905202
+128 6
+129 31335
+130 72
+131 278154
+132 32528
+133 33299431
+134 289515
+135 72241688
+136 628
+137 671628
+138 3798
+139 2945467
+140 728973
+141 64709349
+142 3205057
+143 99730017
+144 19
+145 86296
+146 217
+147 704603
+148 87692
+149 50352721
+150 728318
+151 89072848
+152 1169
+153 1468982
+154 8697
+155 5821059
+156 1602799
+157 82438973
+158 6402424
+159 113177011
+160 22
+161 111200
+162 263
+163 821474
+164 113822
+165 41785539
+166 872073
+167 76702603
+168 734
+169 753231
+170 4143
+171 3512464
+172 809672
+173 71693751
+174 3856651
+175 104344446
+176 67
+177 273672
+178 744
+179 1742709
+180 285398
+181 57385824
+182 1920352
+183 91758819
+184 1332
+185 1689133
+186 9413
+187 6897407
+188 1821788
+189 88978769
+190 7611629
+191 116410794
+192 51
+193 171884
+194 560
+195 1169383
+196 183414
+197 49288586
+198 1305325
+199 85851630
+200 815
+201 842521
+202 4624
+203 4010811
+204 907399
+205 77349281
+206 4354569
+207 109042390
+208 144
+209 415680
+210 1523
+211 2475079
+212 441787
+213 66734849
+214 2864972
+215 100470889
+216 1505
+217 1871044
+218 10528
+219 7754601
+220 2042714
+221 93029295
+222 8834572
+223 117923381
+224 160
+225 454208
+226 1684
+227 2583569
+228 503525
+229 57311428
+230 2989675
+231 90863923
+232 1048
+233 1140000
+234 6050
+235 5382389
+236 1238979
+237 84241270
+238 6191583
+239 114362710
+240 428
+241 972308
+242 4150
+243 4729682
+244 1089252
+245 72065430
+246 5730712
+247 104402615
+248 1939
+249 2462027
+250 13624
+251 10071894
+252 2712730
+253 97601168
+254 11673557
+255 122190940
+256 2
+257 9498
+258 24
+259 79945
+260 10752
+261 14258615
+262 107239
+263 50856457
+264 452
+265 697726
+266 4490
+267 2822825
+268 932822
+269 53639060
+270 3662872
+271 90489596
+272 8
+273 29798
+274 95
+275 223957
+276 41341
+277 28442950
+278 332878
+279 68930124
+280 1699
+281 1531510
+282 13519
+283 5635662
+284 1989662
+285 69678635
+286 6989874
+287 103854924
+288 10
+289 38592
+290 120
+291 295819
+292 51637
+293 31369690
+294 434324
+295 70938987
+296 671
+297 721499
+298 5038
+299 3147896
+300 968177
+301 64784337
+302 4035226
+303 98482649
+304 32
+305 101297
+306 365
+307 712465
+308 143822
+309 48962693
+310 1050839
+311 88025320
+312 2013
+313 1585225
+314 14261
+315 6129197
+316 2060910
+317 81099274
+318 7634802
+319 111395308
+320 27
+321 57497
+322 257
+323 380724
+324 79504
+325 34879820
+326 587405
+327 76210771
+328 734
+329 789662
+330 5397
+331 3524986
+332 1088152
+333 68215167
+334 4691841
+335 104893830
+336 75
+337 143289
+338 703
+339 925065
+340 214788
+341 53155396
+342 1384457
+343 93315403
+344 2163
+345 1777001
+346 15550
+347 6864447
+348 2381043
+349 85018646
+350 8861967
+351 114587784
+352 103
+353 203441
+354 1031
+355 1216511
+356 296217
+357 48144102
+358 1789235
+359 86052906
+360 965
+361 908466
+362 6345
+363 4128921
+364 1268319
+365 76413089
+366 5553302
+367 108727635
+368 273
+369 459035
+370 2568
+371 2479945
+372 676543
+373 65867786
+374 3570437
+375 101630311
+376 2507
+377 1987608
+378 17331
+379 7980360
+380 2679601
+381 92363718
+382 10275128
+383 117942645
+384 12
+385 45711
+386 143
+387 332427
+388 63139
+389 34188123
+390 513289
+391 74310480
+392 1060
+393 744405
+394 6322
+395 3144421
+396 998196
+397 67000938
+398 4092455
+399 102479781
+400 37
+401 112496
+402 405
+403 801759
+404 159812
+405 51596478
+406 1182427
+407 90983246
+408 2029
+409 1581472
+410 14290
+411 6061271
+412 2047019
+413 84461228
+414 7742372
+415 112474685
+416 43
+417 136352
+418 496
+419 902897
+420 198401
+421 43850605
+422 1324526
+423 78791391
+424 1186
+425 829942
+426 6923
+427 3727578
+428 1138486
+429 73609409
+430 4872398
+431 107810620
+432 126
+433 314911
+434 1339
+435 1863528
+436 460075
+437 59860345
+438 2628087
+439 95065035
+440 2255
+441 1797843
+442 15594
+443 7234754
+444 2398192
+445 90382853
+446 9184313
+447 118024610
+448 100
+449 204678
+450 1013
+451 1288671
+452 311368
+453 51461477
+454 1940945
+455 88461722
+456 1319
+457 913332
+458 7819
+459 4144085
+460 1279201
+461 79110460
+462 5635626
+463 110369460
+464 273
+465 477392
+466 2657
+467 2622297
+468 712921
+469 69125337
+470 3764291
+471 101922838
+472 2531
+473 2001102
+474 17156
+475 8011737
+476 2725091
+477 95464996
+478 10557559
+479 121667004
+480 302
+481 514289
+482 2955
+483 2727750
+484 764150
+485 57613471
+486 3864725
+487 91514836
+488 1671
+489 1216475
+490 10000
+491 5622217
+492 1711059
+493 87066187
+494 7699073
+495 115145694
+496 770
+497 1059840
+498 7002
+499 4971895
+500 1543099
+501 73147277
+502 6992897
+503 105256346
+504 3194
+505 2628403
+506 22318
+507 10634614
+508 3581598
+509 99591089
+510 13694936
+511 123805801
+512 97
+513 438583
+514 1082
+515 2709770
+516 467793
+517 62233795
+518 3236365
+519 92864138
+520 873
+521 1201180
+522 4916
+523 5992226
+524 1295951
+525 89881852
+526 6821855
+527 116111968
+528 290
+529 1027455
+530 3181
+531 5372086
+532 1118502
+533 76874973
+534 6477401
+535 106174660
+536 1934
+537 2683421
+538 13540
+539 11305324
+540 2991570
+541 103334266
+542 13295977
+543 124348132
+544 270
+545 1071956
+546 3026
+547 5786650
+548 1181191
+549 67812072
+550 7082943
+551 98196271
+552 1181
+553 1726665
+554 7259
+555 8291732
+556 1877832
+557 93139506
+558 10029949
+559 119823816
+560 789
+561 2302526
+562 8328
+563 9912121
+564 2601558
+565 82274208
+566 12455751
+567 110555733
+568 2644
+569 3732049
+570 19144
+571 15286252
+572 4134722
+573 106709277
+574 18281589
+575 128436771
+576 528
+577 1488216
+578 5508
+579 7357343
+580 1692952
+581 70505609
+582 9163234
+583 99934902
+584 1509
+585 2198656
+586 9779
+587 10127264
+588 2482129
+589 97628745
+590 12274594
+591 122862474
+592 1472
+593 3124699
+594 14405
+595 12289082
+596 3626702
+597 83834130
+598 15663342
+599 112128796
+600 3392
+601 4553823
+602 24896
+603 17313176
+604 5234246
+605 110223356
+606 21393439
+607 129849677
+608 1706
+609 2983727
+610 15499
+611 12215091
+612 3493957
+613 74709369
+614 15524952
+615 102572764
+616 2894
+617 3633552
+618 20698
+619 14875112
+620 4164119
+621 99922065
+622 18698435
+623 125227448
+624 4256
+625 5549352
+626 36184
+627 18803250
+628 6673860
+629 87286090
+630 24874079
+631 115575487
+632 6459
+633 7016394
+634 47556
+635 23976463
+636 8150783
+637 112752491
+638 30924190
+639 133441422
+640 266
+641 1207316
+642 3052
+643 6636130
+644 1300828
+645 69384103
+646 8156139
+647 98959683
+648 1398
+649 1923357
+650 8325
+651 9397687
+652 2082573
+653 95659787
+654 11220516
+655 119557748
+656 796
+657 2638421
+658 8734
+659 11505399
+660 2920808
+661 82860830
+662 14608740
+663 110360653
+664 2664
+665 4175381
+666 19534
+667 16801270
+668 4630930
+669 108806369
+670 20353452
+671 130528080
+672 939
+673 2827946
+674 10260
+675 12029336
+676 3182512
+677 73512155
+678 15772458
+679 102756341
+680 2344
+681 3476032
+682 16025
+683 14821628
+684 3912061
+685 100378233
+686 18580297
+687 124831025
+688 2621
+689 5304217
+690 25764
+691 18836482
+692 6247823
+693 86432297
+694 24530141
+695 115054234
+696 4916
+697 6820121
+698 37208
+699 24204404
+700 7854361
+701 110867598
+702 30209819
+703 135122998
+704 1604
+705 3317097
+706 15294
+707 13997817
+708 3885042
+709 75085038
+710 17901482
+711 103964100
+712 3112
+713 3999443
+714 21348
+715 16609940
+716 4647775
+717 100722007
+718 21008944
+719 127918093
+720 4244
+721 6276421
+722 36834
+723 21104546
+724 7602232
+725 88383830
+726 27954151
+727 116964820
+728 6670
+729 7958519
+730 48705
+731 26741535
+732 9310460
+733 114567332
+734 34012957
+735 137067139
+736 5334
+737 6688120
+738 45994
+739 21913788
+740 8199538
+741 78744048
+742 28615773
+743 108064945
+744 7301
+745 7374315
+746 51827
+747 24569184
+748 8975876
+749 104718633
+750 31172497
+751 131132532
+752 12668
+753 11226730
+754 98340
+755 30834996
+756 14086985
+757 93251462
+758 40453981
+759 120846110
+760 15512
+761 12738469
+762 110887
+763 36298409
+764 15874117
+765 115913270
+766 45644274
+767 138038027
+768 192
+769 494272
+770 2101
+771 2924324
+772 764314
+773 64313489
+774 4326195
+775 97104611
+776 1311
+777 1305415
+778 8457
+779 6165294
+780 1901623
+781 90611558
+782 8669189
+783 118554901
+784 577
+785 1147131
+786 5889
+787 5536938
+788 1721834
+789 78132034
+790 8046126
+791 108884068
+792 3229
+793 2866087
+794 22554
+795 11657118
+796 4013420
+797 105254429
+798 15495349
+799 127470105
+800 538
+801 1181321
+802 5741
+803 6020531
+804 1803671
+805 69690385
+806 8449098
+807 98671282
+808 1814
+809 1877892
+810 12291
+811 8754810
+812 2759541
+813 97772121
+814 12082368
+815 120788386
+816 1528
+817 2450462
+818 15039
+819 10372968
+820 3544857
+821 83722940
+822 14334373
+823 110816273
+824 4330
+825 3972188
+826 31649
+827 15765886
+828 5545733
+829 109234519
+830 20713321
+831 130127499
+832 1002
+833 1577814
+834 9545
+835 7576178
+836 2439206
+837 71766071
+838 10915295
+839 100458148
+840 2343
+841 2328703
+842 16180
+843 10483353
+844 3441733
+845 99813101
+846 14611780
+847 121316162
+848 2715
+849 3241664
+850 23634
+851 12808809
+852 4752410
+853 85834475
+854 17805351
+855 113391766
+856 5600
+857 4757337
+858 40361
+859 18190693
+860 6739328
+861 112781190
+862 24058555
+863 133824858
+864 2874
+865 3159084
+866 24521
+867 12492065
+868 4494122
+869 76201036
+870 17726844
+871 102131562
+872 4475
+873 3832627
+874 32005
+875 15438631
+876 5410669
+877 103092393
+878 21025747
+879 124629007
+880 7014
+881 5815341
+882 54957
+883 19411364
+884 8002622
+885 88589435
+886 26878537
+887 118293969
+888 10339
+889 7377283
+890 72321
+891 24837882
+892 10041432
+893 114001415
+894 33362816
+895 134556926
+896 528
+897 1302023
+898 5874
+899 6916181
+900 2002364
+901 70992688
+902 9738517
+903 99824048
+904 2204
+905 2060794
+906 13912
+907 9725899
+908 2976365
+909 97543583
+910 13193140
+911 121717746
+912 1564
+913 2787402
+914 15871
+915 12101312
+916 3938817
+917 85048610
+918 16807580
+919 113186566
+920 4445
+921 4309356
+922 32492
+923 17342856
+924 5976510
+925 110425493
+926 23151915
+927 129144702
+928 1707
+929 2935631
+930 17601
+931 12430272
+932 4154926
+933 73795002
+934 17277433
+935 104447950
+936 3748
+937 3655594
+938 26258
+939 15269699
+940 5086842
+941 102039383
+942 20774859
+943 125357623
+944 4693
+945 5489921
+946 42290
+947 19427124
+948 7514459
+949 87672236
+950 26451367
+951 115013581
+952 8006
+953 7124350
+954 60434
+955 24593613
+956 9513495
+957 114569527
+958 33288566
+959 134472524
+960 2900
+961 3466345
+962 25122
+963 14315270
+964 4969555
+965 77208126
+966 19818490
+967 105788076
+968 4990
+969 4221110
+970 33871
+971 17255470
+972 5969058
+973 102263401
+974 23737233
+975 128807532
+976 7382
+977 6537423
+978 58610
+979 21948136
+980 8919704
+981 90888626
+982 30390361
+983 119051840
+984 10928
+985 8203126
+986 76521
+987 27633249
+988 11140419
+989 114440012
+990 36992653
+991 135276469
+992 8907
+993 6923754
+994 70349
+995 22454011
+996 9467754
+997 80151964
+998 31126922
+999 110226875
+1000 11535
+1001 7690684
+1002 80056
+1003 25245134
+1004 10480355
+1005 107198218
+1006 34288000
+1007 129185160
+1008 20582
+1009 11741125
+1010 146083
+1011 32064284
+1012 15900178
+1013 93412095
+1014 42446327
+1015 122663267
+1016 24255
+1017 13339271
+1018 166816
+1019 36578306
+1020 18058472
+1021 118691906
+1022 49121357
+1023 140113432
+1024 31
+1025 150413
+1026 366
+1027 1158644
+1028 157542
+1029 51738914
+1030 1251433
+1031 87564031
+1032 872
+1033 842251
+1034 4476
+1035 4113036
+1036 900240
+1037 78408308
+1038 4567962
+1039 111542416
+1040 93
+1041 402568
+1042 1086
+1043 2554935
+1044 431312
+1045 67925585
+1046 2942411
+1047 102916480
+1048 1527
+1049 1896622
+1050 10210
+1051 8192965
+1052 2063099
+1053 93601052
+1054 9265168
+1055 121858401
+1056 88
+1057 370225
+1058 1025
+1059 2417970
+1060 396557
+1061 61259477
+1062 2777133
+1063 93660336
+1064 1045
+1065 1001241
+1066 5311
+1067 5111023
+1068 1082547
+1069 85511233
+1070 5672325
+1071 116016162
+1072 265
+1073 895954
+1074 2848
+1075 4821805
+1076 995465
+1077 75619690
+1078 5751715
+1079 107422928
+1080 1819
+1081 2306325
+1082 11993
+1083 9782219
+1084 2516472
+1085 100548867
+1086 11462175
+1087 125532977
+1088 192
+1089 629408
+1090 1986
+1091 3545722
+1092 706626
+1093 63252380
+1094 4511461
+1095 95363459
+1096 1256
+1097 1406524
+1098 7031
+1099 6598640
+1100 1581968
+1101 89406248
+1102 7921177
+1103 117127451
+1104 546
+1105 1500980
+1106 5318
+1107 6740546
+1108 1748642
+1109 77488092
+1110 8648849
+1111 107904509
+1112 2351
+1113 3158581
+1114 16590
+1115 12308887
+1116 3591894
+1117 103824241
+1118 15157568
+1119 124572802
+1120 528
+1121 1293488
+1122 5225
+1123 6207972
+1124 1503400
+1125 68975282
+1126 7991050
+1127 97278780
+1128 1646
+1129 1936460
+1130 10057
+1131 8990579
+1132 2211374
+1133 93764350
+1134 10894996
+1135 118538464
+1136 1422
+1137 2711838
+1138 13151
+1139 10697510
+1140 3240677
+1141 82089910
+1142 13911964
+1143 112665340
+1144 3216
+1145 4105046
+1146 23032
+1147 15785201
+1148 4793386
+1149 106967458
+1150 19875798
+1151 128446296
+1152 40
+1153 192297
+1154 475
+1155 1396806
+1156 202836
+1157 52637774
+1158 1606679
+1159 89369726
+1160 902
+1161 828488
+1162 4563
+1163 4057633
+1164 892115
+1165 78947682
+1166 4628052
+1167 111422777
+1168 120
+1169 516183
+1170 1391
+1171 3097751
+1172 564017
+1173 69601145
+1174 3637734
+1175 102905271
+1176 1546
+1177 1933969
+1178 10305
+1179 8269340
+1180 2104002
+1181 95781245
+1182 9426626
+1183 120561698
+1184 89
+1185 369612
+1186 1026
+1187 2408796
+1188 397318
+1189 59904945
+1190 2820020
+1191 94660030
+1192 1045
+1193 999782
+1194 5330
+1195 5094626
+1196 1093516
+1197 86313767
+1198 5690937
+1199 114097583
+1200 267
+1201 891221
+1202 2816
+1203 4842290
+1204 987372
+1205 75279605
+1206 5732936
+1207 105857115
+1208 1809
+1209 2307523
+1210 12149
+1211 9964552
+1212 2526404
+1213 100760890
+1214 11414974
+1215 123666639
+1216 274
+1217 755065
+1218 2729
+1219 3977942
+1220 870519
+1221 64659605
+1222 5120758
+1223 96933647
+1224 1278
+1225 1396320
+1226 7119
+1227 6565673
+1228 1557192
+1229 90355168
+1230 7938956
+1231 115782522
+1232 774
+1233 1720285
+1234 7329
+1235 7194896
+1236 2048618
+1237 78967236
+1238 9406784
+1239 107980362
+1240 2353
+1241 3123420
+1242 16670
+1243 12183951
+1244 3583355
+1245 103563328
+1246 15079836
+1247 124974132
+1248 525
+1249 1315885
+1250 5228
+1251 6242417
+1252 1522222
+1253 68808113
+1254 8105522
+1255 97128744
+1256 1671
+1257 1934414
+1258 10089
+1259 8992037
+1260 2195395
+1261 92646810
+1262 10827707
+1263 120784730
+1264 1418
+1265 2700373
+1266 13078
+1267 10786324
+1268 3231141
+1269 83046435
+1270 13975930
+1271 111979610
+1272 3220
+1273 4102509
+1274 23118
+1275 15607046
+1276 4771277
+1277 106799854
+1278 19520816
+1279 128796256
+1280 62
+1281 184096
+1282 708
+1283 1226094
+1284 286421
+1285 53802398
+1286 1996014
+1287 90620362
+1288 1357
+1289 924841
+1290 7488
+1291 4331679
+1292 1321421
+1293 81221055
+1294 6008660
+1295 111894723
+1296 186
+1297 458807
+1298 2043
+1299 2766075
+1300 720141
+1301 71253190
+1302 4155892
+1303 103750216
+1304 2519
+1305 2084785
+1306 17022
+1307 8646435
+1308 2851706
+1309 98202027
+1310 11287111
+1311 122619487
+1312 176
+1313 420756
+1314 1880
+1315 2561851
+1316 659248
+1317 61062843
+1318 3869343
+1319 95103270
+1320 1599
+1321 1108260
+1322 8792
+1323 5326849
+1324 1588410
+1325 87430404
+1326 7470599
+1327 115919794
+1328 505
+1329 981712
+1330 5058
+1331 5136174
+1332 1520407
+1333 78164384
+1334 7319615
+1335 107199735
+1336 2967
+1337 2463767
+1338 20121
+1339 10261048
+1340 3464105
+1341 100962295
+1342 13606407
+1343 122701080
+1344 364
+1345 701750
+1346 3503
+1347 3800034
+1348 1137022
+1349 64795529
+1350 5848348
+1351 96736498
+1352 1994
+1353 1526157
+1354 11745
+1355 6964018
+1356 2305917
+1357 91172102
+1358 9880389
+1359 119077644
+1360 1013
+1361 1622751
+1362 9579
+1363 7019881
+1364 2523931
+1365 80951123
+1366 10428309
+1367 109408699
+1368 3941
+1369 3277940
+1370 27831
+1371 12852475
+1372 4758310
+1373 106022332
+1374 17495641
+1375 126719249
+1376 972
+1377 1399891
+1378 8883
+1379 6522581
+1380 2149418
+1381 69594418
+1382 9537448
+1383 100195532
+1384 2646
+1385 2075924
+1386 16513
+1387 9174104
+1388 3073036
+1389 96008133
+1390 13150409
+1391 120614695
+1392 2544
+1393 2847171
+1394 21947
+1395 11133816
+1396 4270758
+1397 84317788
+1398 15877777
+1399 112157213
+1400 5329
+1401 4394623
+1402 37521
+1403 16359478
+1404 6221391
+1405 108191440
+1406 22252708
+1407 129282760
+1408 80
+1409 233598
+1410 919
+1411 1529897
+1412 372629
+1413 54941648
+1414 2403095
+1415 93082547
+1416 1405
+1417 931537
+1418 7578
+1419 4290188
+1420 1319386
+1421 81636241
+1422 6049584
+1423 111209518
+1424 239
+1425 588606
+1426 2633
+1427 3298108
+1428 944159
+1429 72146846
+1430 4961109
+1431 105176126
+1432 2542
+1433 2085325
+1434 16845
+1435 8528145
+1436 2872494
+1437 97299053
+1438 11408785
+1439 122170761
+1440 175
+1441 428398
+1442 1857
+1443 2577017
+1444 672969
+1445 62750644
+1446 3850272
+1447 93736662
+1448 1602
+1449 1130498
+1450 8813
+1451 5381949
+1452 1619208
+1453 89143817
+1454 7451329
+1455 115301172
+1456 503
+1457 997074
+1458 5019
+1459 5067096
+1460 1515936
+1461 77314474
+1462 7324790
+1463 107872289
+1464 2949
+1465 2504573
+1466 19777
+1467 10264209
+1468 3491214
+1469 101733794
+1470 13764677
+1471 126159667
+1472 519
+1473 828924
+1474 4847
+1475 4166168
+1476 1340404
+1477 66334297
+1478 6439460
+1479 96008951
+1480 2005
+1481 1507150
+1482 11846
+1483 6968586
+1484 2282402
+1485 91127119
+1486 9843793
+1487 118194925
+1488 1424
+1489 1831180
+1490 12770
+1491 7605776
+1492 2883433
+1493 80125968
+1494 11202391
+1495 109486787
+1496 3918
+1497 3307601
+1498 27919
+1499 12955575
+1500 4738709
+1501 103773671
+1502 17275804
+1503 127329438
+1504 969
+1505 1422291
+1506 8909
+1507 6507932
+1508 2178852
+1509 70387439
+1510 9624012
+1511 101004435
+1512 2633
+1513 2073074
+1514 16450
+1515 9257057
+1516 3095246
+1517 96744708
+1518 13164609
+1519 120792891
+1520 2537
+1521 2849173
+1522 21678
+1523 11126753
+1524 4214940
+1525 83496541
+1526 16227929
+1527 111759246
+1528 5282
+1529 4327984
+1530 36901
+1531 16352366
+1532 6228279
+1533 109868812
+1534 22306845
+1535 128783268
+1536 613
+1537 2730648
+1538 7248
+1539 13278299
+1540 3049614
+1541 79907265
+1542 17488876
+1543 107021427
+1544 2039
+1545 3426089
+1546 13111
+1547 15894701
+1548 3779090
+1549 103817282
+1550 20610011
+1551 127619132
+1552 1850
+1553 5750197
+1554 20084
+1555 21326501
+1556 6829385
+1557 90140817
+1558 29179696
+1559 119891574
+1560 4089
+1561 7286104
+1562 31998
+1563 26759490
+1564 8364412
+1565 116915755
+1566 34853541
+1567 135295601
+1568 1947
+1569 5418503
+1570 20958
+1571 20571143
+1572 6344275
+1573 81816479
+1574 27592777
+1575 108579191
+1576 3733
+1577 6083457
+1578 28179
+1579 23099311
+1580 7149218
+1581 108603139
+1582 30825252
+1583 131097109
+1584 5443
+1585 10100080
+1586 53275
+1587 30137763
+1588 12414498
+1589 94443293
+1590 40499460
+1591 122045858
+1592 8237
+1593 11475670
+1594 66003
+1595 35381710
+1596 13907552
+1597 119123389
+1598 47110549
+1599 139272771
+1600 4061
+1601 7127017
+1602 36882
+1603 24401845
+1604 8922688
+1605 84138449
+1606 33420425
+1607 112205558
+1608 5886
+1609 7772691
+1610 44202
+1611 27042921
+1612 9659516
+1613 108984786
+1614 36408983
+1615 130079215
+1616 10558
+1617 12691029
+1618 91335
+1619 34920030
+1620 16478557
+1621 96095309
+1622 46744861
+1623 123508848
+1624 13624
+1625 14030341
+1626 102580
+1627 39459290
+1628 17864280
+1629 121671572
+1630 53134291
+1631 140735841
+1632 11493
+1633 12014204
+1634 95520
+1635 33843551
+1636 15393515
+1637 86890246
+1638 44868182
+1639 114698301
+1640 14164
+1641 12661312
+1642 102269
+1643 37646637
+1644 16290134
+1645 112225670
+1646 49119976
+1647 135335977
+1648 27272
+1649 19243478
+1650 206764
+1651 45579973
+1652 25651324
+1653 98142216
+1654 58063137
+1655 126425142
+1656 31033
+1657 21408777
+1658 224421
+1659 49646834
+1660 27595106
+1661 122883998
+1662 63969490
+1663 144683259
+1664 617
+1665 2679240
+1666 7142
+1667 13283653
+1668 2952028
+1669 80788327
+1670 17722175
+1671 107927325
+1672 2063
+1673 3445371
+1674 13272
+1675 16066069
+1676 3812631
+1677 103282457
+1678 20565602
+1679 125650673
+1680 1844
+1681 5774665
+1682 20286
+1683 21770311
+1684 6799693
+1685 91092592
+1686 28744229
+1687 119392777
+1688 4083
+1689 7211377
+1690 32261
+1691 26442435
+1692 8343834
+1693 115077456
+1694 35234902
+1695 136484876
+1696 1945
+1697 5426386
+1698 21077
+1699 20802936
+1700 6367298
+1701 82046460
+1702 27747469
+1703 110219853
+1704 3779
+1705 6155539
+1706 28029
+1707 23546836
+1708 7168340
+1709 106466176
+1710 30803094
+1711 131112257
+1712 5341
+1713 10034176
+1714 53170
+1715 30473019
+1716 12384347
+1717 94382900
+1718 40983184
+1719 123154627
+1720 8178
+1721 11463619
+1722 66948
+1723 35258213
+1724 13885331
+1725 115631945
+1726 46896987
+1727 138729236
+1728 4027
+1729 7036632
+1730 36665
+1731 24385441
+1732 8812806
+1733 83382945
+1734 33703350
+1735 111467501
+1736 5937
+1737 7981053
+1738 43270
+1739 27665655
+1740 9770689
+1741 108871177
+1742 35993072
+1743 133627510
+1744 10501
+1745 12708057
+1746 91780
+1747 34599617
+1748 16286618
+1749 94619754
+1750 46654440
+1751 124234980
+1752 13604
+1753 14157541
+1754 104051
+1755 40103516
+1756 17871019
+1757 119564746
+1758 51707472
+1759 141461884
+1760 11643
+1761 12067915
+1762 96253
+1763 34014423
+1764 15465732
+1765 86580327
+1766 45226540
+1767 116041041
+1768 13865
+1769 12746628
+1770 103219
+1771 36241695
+1772 16213272
+1773 111309833
+1774 48414814
+1775 135449250
+1776 27000
+1777 19411950
+1778 205500
+1779 44434714
+1780 25714325
+1781 99059044
+1782 57678269
+1783 126866168
+1784 30973
+1785 21102620
+1786 222239
+1787 50330284
+1788 27371928
+1789 122497188
+1790 64093612
+1791 143298155
+1792 1240
+1793 2924331
+1794 13496
+1795 13631199
+1796 4468294
+1797 80097158
+1798 20106155
+1799 108928714
+1800 3282
+1801 3649010
+1802 22380
+1803 16781603
+1804 5388194
+1805 106628403
+1806 23564226
+1807 128033285
+1808 3618
+1809 6021472
+1810 37198
+1811 22156554
+1812 8609761
+1813 93378196
+1814 31350349
+1815 121356005
+1816 6988
+1817 7594121
+1818 54875
+1819 27270994
+1820 10590865
+1821 117257800
+1822 37828233
+1823 136596579
+1824 3646
+1825 5656922
+1826 36404
+1827 21482255
+1828 7994317
+1829 83454123
+1830 30351318
+1831 109286465
+1832 6270
+1833 6425176
+1834 46201
+1835 23962566
+1836 8980191
+1837 107620761
+1838 33626199
+1839 132733059
+1840 9921
+1841 10458528
+1842 88975
+1843 30993855
+1844 14208475
+1845 95250681
+1846 42769354
+1847 123650134
+1848 13975
+1849 11857235
+1850 106420
+1851 36574084
+1852 16257651
+1853 118851270
+1854 50121180
+1855 140997664
+1856 7158
+1857 7372328
+1858 61335
+1859 25408870
+1860 10522563
+1861 84490786
+1862 35395854
+1863 113763173
+1864 9770
+1865 8320083
+1866 70997
+1867 28165783
+1868 11750214
+1869 109864335
+1870 39198261
+1871 132867109
+1872 18628
+1873 13071342
+1874 147439
+1875 35759298
+1876 18399007
+1877 96994118
+1878 49414422
+1879 125529665
+1880 22337
+1881 14802666
+1882 165092
+1883 41444909
+1884 20532132
+1885 121596514
+1886 54883602
+1887 141485763
+1889 12520099
+1890 145714
+1891 35424339
+1892 17663868
+1893 88137185
+1894 47167541
+1895 116264847
+1896 22685
+1897 13224942
+1898 156980
+1899 37818236
+1900 18457151
+1901 113678427
+1902 51664666
+1903 134162002
+1904 44355
+1905 19805951
+1906 310302
+1907 46353216
+1908 28026476
+1909 100495308
+1910 59486731
+1911 127948851
+1912 49058
+1913 21725542
+1914 329641
+1915 51218460
+1916 30049289
+1917 124537971
+1918 66983942
+1919 144770570
+1920 1227
+1921 2928344
+1922 13437
+1923 13616235
+1924 4444041
+1925 80096207
+1926 19749563
+1927 107817260
+1928 3278
+1929 3626551
+1930 22244
+1931 16535803
+1932 5412984
+1933 106282543
+1934 23433780
+1935 126540705
+1936 3683
+1937 6086630
+1938 37432
+1939 22284984
+1940 8580463
+1941 93622551
+1942 31617410
+1943 121838198
+1944 7043
+1945 7673323
+1946 54983
+1947 27831814
+1948 10704915
+1949 117534939
+1950 37519581
+1951 138812008
+1952 3659
+1953 5702609
+1954 36759
+1955 21267091
+1956 8048006
+1957 83274001
+1958 30193501
+1959 110759141
+1960 6207
+1961 6453541
+1962 47072
+1963 24071255
+1964 8976310
+1965 108266609
+1966 33115615
+1967 132031002
+1968 9967
+1969 10220194
+1970 88636
+1971 31213580
+1972 14434863
+1973 96789994
+1974 42471936
+1975 123722116
+1976 14046
+1977 11895366
+1978 106772
+1979 36685350
+1980 16121825
+1981 118751144
+1982 49926109
+1983 140689183
+1984 7189
+1985 7527179
+1986 61452
+1987 25329724
+1988 10616607
+1989 84677617
+1990 36118985
+1991 112532106
+1992 9835
+1993 8251387
+1994 71885
+1995 28035767
+1996 11715520
+1997 110480934
+1998 38941603
+1999 131575707
+2000 18790
+2001 13144752
+2002 146177
+2003 36022195
+2004 18497284
+2005 97131033
+2006 48509887
+2007 125631086
+2008 22669
+2009 14660464
+2010 164199
+2011 40959811
+2012 20388574
+2013 121600974
+2014 55479463
+2015 143099953
+2016 19157
+2017 12698361
+2018 148484
+2019 34577209
+2020 17790461
+2021 87213355
+2022 47476392
+2023 115442622
+2024 22629
+2025 13179261
+2026 154089
+2027 37538204
+2028 18377556
+2029 114492629
+2030 50854718
+2031 136212446
+2032 44950
+2033 20224756
+2034 313127
+2035 46192224
+2036 28234029
+2037 99809422
+2038 60251012
+2039 128322426
+2040 48643
+2041 21659921
+2042 333423
+2043 51327747
+2044 29698685
+2045 125287720
+2046 65541052
+2047 143363712
+2048 7442
+2049 16865576
+2050 86014
+2051 58493528
+2052 11942184
+2053 89632581
+2054 33477680
+2055 114775367
+2056 963389
+2057 62981401
+2058 4060470
+2059 106535613
+2060 36534798
+2061 117901650
+2062 61611213
+2063 136609704
+2064 28157
+2065 32701475
+2066 266899
+2067 80002045
+2068 20605052
+2069 103283230
+2070 44954693
+2071 126825626
+2072 2250088
+2073 83828002
+2074 8204614
+2075 122431797
+2076 48724593
+2077 127569021
+2078 74101651
+2079 147219073
+2080 36778
+2081 36023749
+2082 365157
+2083 80885359
+2084 22836215
+2085 102704007
+2086 46195873
+2087 124924944
+2088 1007977
+2089 73640279
+2090 4400657
+2091 116541566
+2092 42983632
+2093 123015751
+2094 68629830
+2095 141542925
+2096 107596
+2097 55596098
+2098 928674
+2099 100144751
+2100 32428219
+2101 114069321
+2102 56315717
+2103 132896191
+2104 2301083
+2105 95488907
+2106 8992628
+2107 129755866
+2108 55800262
+2109 133664018
+2110 81270095
+2111 148867980
+2112 60818
+2113 39478388
+2114 496621
+2115 87346550
+2116 24864797
+2117 104142521
+2118 49324042
+2119 127890693
+2120 1085707
+2121 80954470
+2122 5077239
+2123 120109435
+2124 46621404
+2125 125315012
+2126 72216620
+2127 144409677
+2128 172981
+2129 58845074
+2130 1246896
+2131 104330868
+2132 35009260
+2133 116873522
+2134 59436855
+2135 135497858
+2136 2545438
+2137 100546572
+2138 10140144
+2139 135715986
+2140 59536760
+2141 135964268
+2142 85499041
+2143 150029514
+2144 238274
+2145 54642555
+2146 1714172
+2147 98350715
+2148 33013034
+2149 113006861
+2150 56919070
+2151 131203018
+2152 1251207
+2153 89232062
+2154 5957049
+2155 126359432
+2156 51413519
+2157 129235200
+2158 76044270
+2159 144135646
+2160 582156
+2161 75540121
+2162 3578468
+2163 115717529
+2164 43538840
+2165 122547316
+2166 66733129
+2167 138354560
+2168 2911859
+2169 108527925
+2170 11952963
+2171 142266931
+2172 63800323
+2173 140744598
+2174 89649673
+2175 152470976
+2176 43383
+2177 39263661
+2178 411002
+2179 84492190
+2180 24470845
+2181 102456000
+2182 47457739
+2183 126078295
+2184 1013394
+2185 77029197
+2186 4471597
+2187 118855872
+2188 45135873
+2189 125474980
+2190 69892571
+2191 143710048
+2192 115293
+2193 59576930
+2194 995537
+2195 103955764
+2196 34743010
+2197 116341158
+2198 58239452
+2199 136542962
+2200 2302542
+2201 98978702
+2202 8995018
+2203 134284290
+2204 56939156
+2205 134830244
+2206 82705434
+2207 148793193
+2208 150765
+2209 48730052
+2210 1230270
+2211 90512132
+2212 29259164
+2213 107583767
+2214 51446486
+2215 128433785
+2216 1150230
+2217 86816160
+2218 5371747
+2219 123057095
+2220 49396563
+2221 129127656
+2222 74946901
+2223 145933533
+2224 376709
+2225 67879848
+2226 2621339
+2227 106379416
+2228 38951539
+2229 119842421
+2230 62513965
+2231 135958284
+2232 2553200
+2233 106883629
+2234 10619260
+2235 140153140
+2236 61563673
+2237 137448424
+2238 87265684
+2239 149279657
+2240 239491
+2241 57218391
+2242 1766295
+2243 100559934
+2244 34305465
+2245 113606780
+2246 56959277
+2247 132035577
+2248 1261494
+2249 94129845
+2250 5967315
+2251 128412455
+2252 53604191
+2253 129929026
+2254 77263888
+2255 147354036
+2256 583224
+2257 76647238
+2258 3802573
+2259 115801104
+2260 45163854
+2261 123675005
+2262 68379476
+2263 138755787
+2264 2899890
+2265 112476388
+2266 12054541
+2267 141710858
+2268 65930021
+2269 141958107
+2270 91062705
+2271 154106165
+2272 653772
+2273 65588601
+2274 4109532
+2275 104064264
+2276 39061480
+2277 117016055
+2278 63426813
+2279 135467580
+2280 1676456
+2281 99519583
+2282 8403088
+2283 133865809
+2284 57083022
+2285 132631333
+2286 82256852
+2287 149832018
+2288 1424066
+2289 83066825
+2290 7765505
+2291 120475696
+2292 49679924
+2293 126938414
+2294 74105398
+2295 140801362
+2296 3718454
+2297 119135433
+2298 15946010
+2299 146224736
+2300 70148221
+2301 142104933
+2302 96908839
+2303 154958056
+2304 14668
+2305 17710389
+2306 152339
+2307 61146735
+2308 13732533
+2309 90835174
+2310 35444430
+2311 116466781
+2312 1260616
+2313 64410707
+2314 4937216
+2315 108449199
+2316 38277941
+2317 117443097
+2318 64843607
+2319 138452662
+2320 52176
+2321 34204242
+2322 436722
+2323 82606724
+2324 22573114
+2325 103729882
+2326 47080639
+2327 126436720
+2328 2682015
+2329 85118611
+2330 9435654
+2331 125944194
+2332 50647240
+2333 129778613
+2334 76996849
+2335 146263652
+2336 69377
+2337 37104209
+2338 588663
+2339 83520402
+2340 24475142
+2341 103244272
+2342 47687296
+2343 127559194
+2344 1331598
+2345 75930140
+2346 5415786
+2347 117728924
+2348 45678017
+2349 124174235
+2350 71183471
+2351 143058177
+2352 186071
+2353 56437806
+2354 1367660
+2355 103700761
+2356 34386372
+2357 117161023
+2358 59736175
+2359 133747983
+2360 2858417
+2361 97791974
+2362 10391163
+2363 131898874
+2364 58101536
+2365 132432762
+2366 82522443
+2367 151687672
+2368 104359
+2369 41456194
+2370 790346
+2371 88686821
+2372 26691937
+2373 107021017
+2374 50531362
+2375 127309226
+2376 1461417
+2377 82359184
+2378 6191439
+2379 120544277
+2380 48268200
+2381 128982575
+2382 74479582
+2383 143907328
+2384 278912
+2385 62650843
+2386 1788613
+2387 109484800
+2388 37529899
+2389 117701107
+2390 61337841
+2391 136051612
+2392 3231099
+2393 103832323
+2394 11985510
+2395 136039587
+2396 62632023
+2397 135123916
+2398 86208609
+2399 148586372
+2400 376683
+2401 55664253
+2402 2317046
+2403 99899322
+2404 34438393
+2405 112967205
+2406 57994305
+2407 131627917
+2408 1676250
+2409 90463878
+2410 7227855
+2411 127804117
+2412 53805431
+2413 132103563
+2414 78104919
+2415 143270132
+2416 850466
+2417 76920281
+2418 4586905
+2419 116840045
+2420 45485855
+2421 125698265
+2422 69907656
+2423 139989131
+2424 3591859
+2425 112044567
+2426 13888317
+2427 142815326
+2428 66150096
+2429 139567154
+2430 90548743
+2431 150276135
+2432 81959
+2433 40935625
+2434 685066
+2435 85795329
+2436 26541294
+2437 105937421
+2438 49369198
+2439 125730879
+2440 1355339
+2441 80048870
+2442 5384137
+2443 121983603
+2444 47467062
+2445 125698043
+2446 71882909
+2447 140770178
+2448 203883
+2449 61333524
+2450 1543202
+2451 106667809
+2452 36254418
+2453 117017362
+2454 60228564
+2455 136240589
+2456 2854305
+2457 102273023
+2458 10466497
+2459 134877724
+2460 59565228
+2461 137242153
+2462 84018620
+2463 150521373
+2464 256442
+2465 50477471
+2466 1753857
+2467 91888278
+2468 30925017
+2469 108011562
+2470 54282398
+2471 130166879
+2472 1508218
+2473 89534853
+2474 6544535
+2475 126685269
+2476 51444446
+2477 130666832
+2478 75895272
+2479 141711293
+2480 592086
+2481 68911784
+2482 3429824
+2483 109141967
+2484 40924396
+2485 120240847
+2486 65368053
+2487 135068011
+2488 3247228
+2489 108343400
+2490 12296645
+2491 139200103
+2492 64253946
+2493 139464434
+2494 90144018
+2495 150933892
+2496 387203
+2497 60112680
+2498 2471124
+2499 101364240
+2500 36060090
+2501 113115258
+2502 59092846
+2503 133660255
+2504 1688067
+2505 95166867
+2506 7464938
+2507 130201426
+2508 55435813
+2509 131378360
+2510 80339270
+2511 144343901
+2512 891695
+2513 79403496
+2514 4846349
+2515 117591013
+2516 46701564
+2517 127372321
+2518 71594029
+2519 139752761
+2520 3585107
+2521 114633188
+2522 14060404
+2523 142634270
+2524 68291733
+2525 141838947
+2526 92319322
+2527 154201069
+2528 930345
+2529 66936301
+2530 5057724
+2531 108112438
+2532 41230837
+2533 117236757
+2534 63114220
+2535 134889085
+2536 2264842
+2537 102766997
+2538 10189124
+2539 134349467
+2540 59607139
+2541 136132248
+2542 84365276
+2543 145396604
+2544 1934988
+2545 87496829
+2546 8963586
+2547 123821148
+2548 51594867
+2549 126583574
+2550 75453026
+2551 143429208
+2552 4688291
+2553 121133074
+2554 18369851
+2555 146651284
+2556 71294421
+2557 144513528
+2558 98980071
+2559 156474703
+2560 574681
+2561 72013501
+2562 4170731
+2563 108740256
+2564 42160291
+2565 118802622
+2566 65199657
+2567 134318813
+2568 1711436
+2569 106364466
+2570 9377548
+2571 136089932
+2572 61781874
+2573 136063027
+2574 85955728
+2575 149722927
+2576 1460336
+2577 90368418
+2578 8479930
+2579 123461144
+2580 52938120
+2581 127696495
+2582 77264195
+2583 140512222
+2584 4006765
+2585 123742005
+2586 17570016
+2587 149293906
+2588 74254766
+2589 142980282
+2590 99639510
+2591 155578996
+2592 1511044
+2593 78158991
+2594 9250811
+2595 113168463
+2596 47728382
+2597 121344153
+2598 72033721
+2599 140215256
+2600 2570305
+2601 111183745
+2602 13333752
+2603 139295086
+2604 65791739
+2605 139747945
+2606 91745662
+2607 150998682
+2608 3444869
+2609 95018873
+2610 16736273
+2611 129851414
+2612 59218692
+2613 131000976
+2614 84000432
+2615 145265002
+2616 5672362
+2617 129042313
+2618 24831279
+2619 154064778
+2620 79250112
+2621 148089275
+2622 105806379
+2623 159746592
+2624 2109900
+2625 81321389
+2626 11843659
+2627 115581441
+2628 51249627
+2629 121796815
+2630 74697826
+2631 140540844
+2632 3240333
+2633 114853705
+2634 16467288
+2635 141781340
+2636 68579437
+2637 140809504
+2638 94904134
+2639 155028084
+2640 4586544
+2641 99753752
+2642 20819775
+2643 130425050
+2644 63177958
+2645 133283733
+2646 89252057
+2647 148817311
+2648 7053685
+2649 132802587
+2650 29207405
+2651 157030973
+2652 82807923
+2653 147127777
+2654 108975046
+2655 159194852
+2656 4445525
+2657 87538245
+2658 20784213
+2659 118896880
+2660 57340871
+2661 126578306
+2662 82097782
+2663 143862076
+2664 5584146
+2665 118805371
+2666 25221541
+2667 147671549
+2668 74271136
+2669 143730494
+2670 101333431
+2671 159741857
+2672 8851766
+2673 106132315
+2674 33077565
+2675 135998977
+2676 69673935
+2677 138169687
+2678 97158858
+2679 152019357
+2680 11056251
+2681 137568589
+2682 41686204
+2683 158704164
+2684 88686482
+2685 153232585
+2686 119340173
+2687 166854230
+2688 1636939
+2689 79441694
+2690 10758207
+2691 114454589
+2692 48997170
+2693 121025473
+2694 73788585
+2695 140017108
+2696 2803974
+2697 113091943
+2698 15035959
+2699 143213303
+2700 67542300
+2701 139865148
+2702 94357868
+2703 151251593
+2704 3949896
+2705 96756463
+2706 19757814
+2707 127100039
+2708 61307563
+2709 133125189
+2710 86228962
+2711 148212827
+2712 6212546
+2713 131017681
+2714 27930582
+2715 151134460
+2716 82373955
+2717 146627678
+2718 108157839
+2719 161615558
+2720 4214631
+2721 85948560
+2722 20705208
+2723 119264661
+2724 55552169
+2725 124259355
+2726 81871855
+2727 142802420
+2728 5277704
+2729 118871743
+2730 25065926
+2731 145881513
+2732 73629648
+2733 144891803
+2734 101051234
+2735 156434054
+2736 8345645
+2737 103642148
+2738 33131740
+2739 135995086
+2740 69281917
+2741 139691917
+2742 95699348
+2743 155369212
+2744 10648845
+2745 137233318
+2746 41054643
+2747 161076108
+2748 89238030
+2749 153490810
+2750 116768211
+2751 165921864
+2752 4930729
+2753 89048703
+2754 23605140
+2755 120634134
+2756 58062708
+2757 128107941
+2758 83936018
+2759 144465779
+2760 6018705
+2761 121923012
+2762 27889312
+2763 146495022
+2764 77138497
+2765 144984094
+2766 104193266
+2767 157293335
+2768 10010204
+2769 104420755
+2770 37849428
+2771 138752421
+2772 72704805
+2773 139749859
+2774 100078206
+2775 154595744
+2776 12221109
+2777 139139719
+2778 45853920
+2779 162191363
+2780 92493968
+2781 155114895
+2782 120804768
+2783 166649637
+2784 10508975
+2785 95266740
+2786 37787523
+2787 126247609
+2788 67366579
+2789 131265716
+2790 94198384
+2791 149777177
+2792 11574993
+2793 127074826
+2794 42907803
+2795 154953176
+2796 85512056
+2797 149108134
+2798 112059991
+2799 163299205
+2800 18591730
+2801 113565050
+2802 54596835
+2803 142248590
+2804 82610665
+2805 147555374
+2806 109980262
+2807 161349248
+2808 20935997
+2809 146140513
+2810 62193161
+2811 167341203
+2812 100511596
+2813 161199461
+2814 132696320
+2815 173113372
+2816 932098
+2817 73705793
+2818 5502116
+2819 109501785
+2820 43830184
+2821 119884101
+2822 67377838
+2823 135458192
+2824 2433685
+2825 108299931
+2826 11187411
+2827 138677489
+2828 63177309
+2829 135039974
+2830 88575169
+2831 148462632
+2832 2099186
+2833 91644863
+2834 10402545
+2835 123438153
+2836 55789068
+2837 128257948
+2838 79511331
+2839 143224007
+2840 5180676
+2841 126794604
+2842 20185085
+2843 149019808
+2844 75889862
+2845 147076860
+2846 100530181
+2847 156047939
+2848 2189646
+2849 80418880
+2850 10974090
+2851 114024031
+2852 49385284
+2853 120984582
+2854 73806199
+2855 137143764
+2856 3487905
+2857 113058554
+2858 15835036
+2859 142433293
+2860 67052820
+2861 139186514
+2862 93483191
+2863 152104699
+2864 4460768
+2865 97221285
+2866 18890399
+2867 129429882
+2868 60588477
+2869 131574867
+2870 85670376
+2871 147281730
+2872 7220349
+2873 131534709
+2874 27590271
+2875 154379106
+2876 82133206
+2877 146742055
+2878 107964112
+2879 160865797
+2880 2918655
+2881 83271649
+2882 13792791
+2883 116183089
+2884 52800211
+2885 124907760
+2886 77753669
+2887 141152439
+2888 4324336
+2889 116738096
+2890 18630462
+2891 144629016
+2892 70294035
+2893 139799710
+2894 97193310
+2895 152565537
+2896 5872337
+2897 100103262
+2898 23077597
+2899 131420534
+2900 63762812
+2901 134538502
+2902 89113924
+2903 149626971
+2904 8898348
+2905 133046197
+2906 31919716
+2907 155554649
+2908 84411661
+2909 150932008
+2910 110315168
+2911 160806197
+2912 5586151
+2913 88394234
+2914 22724402
+2915 120775622
+2916 58559218
+2917 125781349
+2918 83317030
+2919 144643371
+2920 6824151
+2921 119523033
+2922 27518088
+2923 147717490
+2924 74961984
+2925 143160327
+2926 104434713
+2927 154217660
+2928 10316210
+2929 106559793
+2930 34746065
+2931 137142638
+2932 72468402
+2933 138578200
+2934 97381392
+2935 152493106
+2936 13102378
+2937 138696090
+2938 44108163
+2939 158160891
+2940 91667921
+2941 155595728
+2942 119431285
+2943 169130090
+2944 2459776
+2945 82018742
+2946 12619666
+2947 114673305
+2948 51264060
+2949 121450095
+2950 75216592
+2951 140219369
+2952 3832509
+2953 114922638
+2954 17685242
+2955 142659030
+2956 69266767
+2957 139779462
+2958 94511031
+2959 153169181
+2960 5075213
+2961 98753536
+2962 21902697
+2963 130770037
+2964 62594333
+2965 133307792
+2966 88148296
+2967 149733699
+2968 7890118
+2969 133358775
+2970 30457713
+2971 154517030
+2972 82694085
+2973 149085761
+2974 110386899
+2975 162252240
+2976 5259600
+2977 87806906
+2978 22507445
+2979 119466498
+2980 57691083
+2981 127707815
+2982 82756572
+2983 144327024
+2984 6629992
+2985 120842226
+2986 27524276
+2987 149509918
+2988 75671990
+2989 143701181
+2990 101564735
+2991 158538548
+2992 9864960
+2993 105532019
+2994 35020301
+2995 134985234
+2996 71764499
+2997 139833958
+2998 98352927
+2999 155659614
+3001 139895979
+3002 44146910
+3003 160456800
+3004 91124233
+3005 154702227
+3006 118797971
+3007 165294928
+3008 6120589
+3009 89990599
+3010 26066681
+3011 121371393
+3012 59895248
+3013 128435485
+3014 85609465
+3015 145632698
+3016 7508838
+3017 124043698
+3018 30719698
+3019 148239576
+3020 78652923
+3021 145656165
+3022 105904822
+3023 158329638
+3024 11695663
+3025 108843684
+3026 40286674
+3027 139249337
+3028 74942420
+3029 140565207
+3030 102277201
+3031 155999482
+3032 14623104
+3033 141133804
+3034 48692746
+3035 163882184
+3036 95294425
+3037 156150093
+3038 121729991
+3039 167797321
+3040 12291581
+3041 97604203
+3042 40110999
+3043 127700624
+3044 68699258
+3045 134376636
+3046 94907005
+3047 150694435
+3048 13187179
+3049 129803067
+3050 44804540
+3051 156295692
+3052 86093792
+3053 150835451
+3054 114573263
+3055 162880486
+3056 20622931
+3057 113878196
+3058 56577971
+3059 145053366
+3060 84172186
+3061 145584375
+3062 110457617
+3063 161543161
+3064 23289574
+3065 148412254
+3066 63970562
+3067 169267050
+3068 103908901
+3069 161167731
+3070 131329206
+3071 175230378
+3072 201255
+3073 59731483
+3074 1689145
+3075 101838541
+3076 35442161
+3077 115374765
+3078 59385205
+3079 132757681
+3080 1249145
+3081 93840481
+3082 6199928
+3083 131414132
+3084 53647983
+3085 131373452
+3086 78174796
+3087 148119850
+3088 542664
+3089 79621263
+3090 3876917
+3091 119105159
+3092 46586724
+3093 123445850
+3094 69698852
+3095 138814459
+3096 2850905
+3097 114957027
+3098 12531555
+3099 144048258
+3100 66681331
+3101 139011208
+3102 91800004
+3103 154175942
+3104 501443
+3105 69259471
+3106 3722407
+3107 106654913
+3108 40997621
+3109 118200849
+3110 64188238
+3111 137675699
+3112 1486697
+3113 102261156
+3114 7674235
+3115 134328787
+3116 58868707
+3117 134984447
+3118 83082004
+3119 146802999
+3120 1265809
+3121 88129789
+3122 7583345
+3123 122704229
+3124 51159928
+3125 126733648
+3126 75364346
+3127 142255862
+3128 3467852
+3129 121589303
+3130 15482688
+3131 147039130
+3132 71066829
+3133 143951449
+3134 96175217
+3135 153034960
+3136 861281
+3137 72392234
+3138 5834356
+3139 108865563
+3140 43268392
+3141 118524644
+3142 66502263
+3143 137807945
+3144 2045323
+3145 104683309
+3146 10566776
+3147 137770224
+3148 60762343
+3149 136609754
+3150 85098922
+3151 150234894
+3152 2140794
+3153 91492088
+3154 11452116
+3155 124736891
+3156 54407860
+3157 129707429
+3158 79445660
+3159 143893689
+3160 4684592
+3161 123585345
+3162 20155767
+3163 147059328
+3164 74657088
+3165 145920539
+3166 99082967
+3167 157018711
+3168 1874244
+3169 79482608
+3170 10525107
+3171 114509845
+3172 47568341
+3173 121695906
+3174 73006780
+3175 139854248
+3176 2865098
+3177 112490058
+3178 14492926
+3179 138201457
+3180 65175254
+3181 139228334
+3182 89368342
+3183 151098033
+3184 4027295
+3185 96408561
+3186 18317287
+3187 128917773
+3188 60302029
+3189 132214740
+3190 85437184
+3191 147411661
+3192 6261812
+3193 129089675
+3194 26541042
+3195 153457383
+3196 79407235
+3197 147375924
+3198 105540174
+3199 158537902
+3200 254560
+3201 61140942
+3202 2079586
+3203 102273493
+3204 36013453
+3205 114946694
+3206 59477294
+3207 132402234
+3208 1267132
+3209 94864402
+3210 6231016
+3211 128860857
+3212 55510689
+3213 134523471
+3214 79469469
+3215 144267066
+3216 689334
+3217 80999563
+3218 4741176
+3219 120298631
+3220 46729942
+3221 124911553
+3222 71283833
+3223 142858323
+3224 2917360
+3225 113050357
+3226 12637530
+3227 143188891
+3228 65947496
+3229 141674174
+3230 91921576
+3231 149943922
+3232 510696
+3233 69603870
+3234 3712297
+3235 107292850
+3236 40932153
+3237 119204847
+3238 63187269
+3239 134164175
+3240 1487970
+3241 102514083
+3242 7748745
+3243 133204930
+3244 58322635
+3245 135192943
+3246 83430319
+3247 146396759
+3248 1232255
+3249 87574417
+3250 7651678
+3251 122299134
+3252 51608729
+3253 128652865
+3254 75676785
+3255 142815163
+3256 3468854
+3257 117844144
+3258 15560264
+3259 147203671
+3260 71250379
+3261 143672438
+3262 96002950
+3263 155727097
+3264 1048655
+3265 74155079
+3266 6499011
+3267 110034194
+3268 43914601
+3269 118867445
+3270 67227351
+3271 135365814
+3272 2023563
+3273 105774570
+3274 10504641
+3275 135348409
+3276 60849890
+3277 135709282
+3278 86824366
+3279 150693829
+3280 2469269
+3281 90725508
+3282 12271089
+3283 124976308
+3284 55447798
+3285 129375506
+3286 79498401
+3287 144975788
+3288 4690572
+3289 123486373
+3290 19895041
+3291 149404708
+3292 75273636
+3293 147424425
+3294 101025328
+3295 156058249
+3296 1821572
+3297 79106703
+3298 10467740
+3299 112424288
+3300 48199031
+3301 120802354
+3302 71647258
+3303 138326952
+3304 2832315
+3305 111856440
+3306 14662727
+3307 141361472
+3308 64370479
+3309 139523791
+3310 90738322
+3311 150851981
+3312 4065794
+3313 98065847
+3314 18544144
+3315 129932639
+3316 60550138
+3317 132351196
+3318 85055106
+3319 147011736
+3320 6176123
+3321 127322362
+3322 26055854
+3323 151150982
+3324 78358984
+3325 149599777
+3326 106664741
+3327 161191718
+3328 350593
+3329 62615146
+3330 2475087
+3331 105515755
+3332 37175376
+3333 115242950
+3334 60368793
+3335 132396982
+3336 1699513
+3337 95617775
+3338 7787453
+3339 133223916
+3340 56259828
+3341 133484737
+3342 81377286
+3343 149232501
+3344 858869
+3345 82127696
+3346 5179638
+3347 120611324
+3348 47840005
+3349 125486557
+3350 71196706
+3351 139346777
+3352 3698770
+3353 117057797
+3354 15010389
+3355 145651686
+3356 68464703
+3357 142337481
+3358 94123522
+3359 152664625
+3360 810868
+3361 71435120
+3362 4884725
+3363 108137227
+3364 42669279
+3365 118935892
+3366 65940242
+3367 133387634
+3368 2087068
+3369 103438507
+3370 9642219
+3371 134688226
+3372 59815905
+3373 136729236
+3374 84315427
+3375 148160277
+3376 1834089
+3377 90884068
+3378 9333392
+3379 124007072
+3380 53305703
+3381 128876427
+3382 77547121
+3383 143821763
+3384 4505074
+3385 123088299
+3386 17648934
+3387 146362250
+3388 73780102
+3389 143504508
+3390 98199359
+3391 157873555
+3392 1337048
+3393 74881950
+3394 7264025
+3395 109793559
+3396 45830866
+3397 120901486
+3398 67681752
+3399 138388219
+3400 2816706
+3401 107402506
+3402 12765570
+3403 137283378
+3404 62752866
+3405 138156137
+3406 88580702
+3407 148944346
+3408 2987142
+3409 92819040
+3410 13096788
+3411 127400556
+3412 56565901
+3413 130756832
+3414 79908377
+3415 143567227
+3416 6082836
+3417 124942291
+3418 22532602
+3419 151346490
+3420 76952048
+3421 144749367
+3422 102343617
+3423 160225164
+3424 2627108
+3425 79629865
+3426 12164042
+3427 115416910
+3428 49005476
+3429 120791323
+3430 74641972
+3431 141840413
+3432 3821445
+3433 112556854
+3434 16913159
+3435 142643053
+3436 67970843
+3437 139957281
+3438 92610573
+3439 152762121
+3440 5183868
+3441 99146577
+3442 20499242
+3443 130410802
+3444 62158814
+3445 131570277
+3446 87977786
+3447 146079192
+3448 7860273
+3449 131888627
+3450 29380094
+3451 153231401
+3452 82532063
+3453 149181432
+3454 108077797
+3455 160061719
+3456 448558
+3457 63997628
+3458 3051379
+3459 106948567
+3460 38635225
+3461 116580574
+3462 62407879
+3463 133515235
+3464 1708070
+3465 95151317
+3466 7819865
+3467 131643934
+3468 55533763
+3469 131034853
+3470 81019985
+3471 149347855
+3472 1089455
+3473 84404068
+3474 6209900
+3475 120399838
+3476 48719388
+3477 128661147
+3478 73019135
+3479 141018145
+3480 3707767
+3481 116847386
+3482 14903533
+3483 143973354
+3484 69185726
+3485 142333567
+3486 94771167
+3487 150723344
+3488 817452
+3489 71396363
+3490 4931212
+3491 110286309
+3492 42883103
+3493 119084740
+3494 64387696
+3495 136636787
+3496 2055186
+3497 103627193
+3498 9500925
+3499 135086433
+3500 60041609
+3501 134632091
+3502 84624888
+3503 148490588
+3504 1858365
+3505 89686444
+3506 9305349
+3507 124344509
+3508 53413734
+3509 128872221
+3510 78558083
+3511 142207216
+3512 4491162
+3513 121922352
+3514 17784671
+3515 148211704
+3516 72578771
+3517 146768726
+3518 97314728
+3519 157049318
+3520 1580102
+3521 75159932
+3522 7962971
+3523 111114388
+3524 45461001
+3525 120296287
+3526 68775879
+3527 137697161
+3528 2869451
+3529 107741463
+3530 12607076
+3531 137331688
+3532 62805277
+3533 138370160
+3534 88923323
+3535 150257590
+3536 3335540
+3537 93223854
+3538 14000270
+3539 127710180
+3540 56658490
+3541 129786341
+3542 81952551
+3543 143096923
+3544 6000400
+3545 125142371
+3546 22581786
+3547 149624640
+3548 76611832
+3549 145766276
+3550 102217662
+3551 157994443
+3552 2575183
+3553 80831058
+3554 12200074
+3555 114103951
+3556 49804376
+3557 124017676
+3558 73639089
+3559 140005200
+3560 3884995
+3561 114574972
+3562 16723532
+3563 141109021
+3564 66556403
+3565 140322656
+3566 93046065
+3567 152324952
+3568 5168347
+3569 97021723
+3570 20559333
+3571 131608906
+3572 61290469
+3573 133579343
+3574 86967466
+3575 148169247
+3576 7875825
+3577 130424793
+3578 29377930
+3579 152507544
+3580 81322755
+3581 149718803
+3582 109354073
+3583 160127036
+3584 3726135
+3585 91888578
+3586 22805448
+3587 122842903
+3588 59706806
+3589 130232409
+3590 87703611
+3591 142673584
+3592 4872154
+3593 122704994
+3594 26785682
+3595 149257786
+3596 78509420
+3597 145061395
+3598 103470785
+3599 158630687
+3600 8794472
+3601 111443442
+3602 38080921
+3603 139507784
+3604 74551970
+3605 141079438
+3606 101812836
+3607 156351010
+3608 11476360
+3609 142926106
+3610 46322583
+3611 161946524
+3612 94719905
+3613 157007824
+3614 121211705
+3615 165873804
+3616 8188726
+3617 99290499
+3618 36625361
+3619 127939220
+3620 68270127
+3621 133202894
+3622 93523749
+3623 150951776
+3624 9182722
+3625 128432957
+3626 40984717
+3627 153625950
+3628 84549231
+3629 150253640
+3630 111900940
+3631 161050892
+3632 16209191
+3633 114985361
+3634 54419135
+3635 147726743
+3636 83483363
+3637 146137758
+3638 111762010
+3639 161710373
+3640 18336638
+3641 148728085
+3642 63205736
+3643 168326815
+3644 102105951
+3645 160896705
+3646 132399544
+3647 174609400
+3648 10714844
+3649 101869528
+3650 42693587
+3651 130497073
+3652 72196740
+3653 132499698
+3654 98929648
+3655 153139228
+3656 11926795
+3657 131041879
+3658 47612221
+3659 156795719
+3660 88459247
+3661 150765295
+3662 117091664
+3663 165541260
+3664 21111486
+3665 119884532
+3666 61555467
+3667 147577354
+3668 89493423
+3669 149441682
+3670 116510728
+3671 164060933
+3672 23041575
+3673 152359899
+3674 71426527
+3675 169450215
+3676 107145617
+3677 163121178
+3678 136452524
+3679 177584037
+3680 19328205
+3681 107805060
+3682 59719884
+3683 136212434
+3684 80650901
+3685 138835798
+3686 108352478
+3687 157489665
+3688 20802749
+3689 138228385
+3690 63967180
+3691 162472890
+3692 98479953
+3693 155841859
+3694 127950472
+3695 172055482
+3696 33743981
+3697 127576808
+3698 78051912
+3699 154280783
+3700 99061038
+3701 155764227
+3702 125692192
+3703 169270871
+3704 35480592
+3705 157438506
+3706 86461028
+3707 178810281
+3708 118409088
+3709 170976241
+3710 145852243
+3711 183822189
+3712 3735859
+3713 91686737
+3714 22895422
+3715 123688127
+3716 59775111
+3717 127487333
+3718 86088421
+3719 145517426
+3720 4898731
+3721 123414169
+3722 26582530
+3723 150339150
+3724 75899398
+3725 146300399
+3726 104432363
+3727 157670963
+3728 8855984
+3729 110673280
+3730 39149334
+3731 139460356
+3732 74793328
+3733 138165158
+3734 101512017
+3735 158630201
+3736 11141718
+3737 142118103
+3738 47434191
+3739 162686098
+3740 93208144
+3741 155014052
+3742 124793588
+3743 169404304
+3744 8186836
+3745 97763296
+3746 36170716
+3747 128470561
+3748 67106265
+3749 132354207
+3750 95264332
+3751 150705101
+3752 9231275
+3753 129105020
+3754 40848945
+3755 153322838
+3756 85578785
+3757 148962913
+3758 111322308
+3759 159842536
+3760 16372892
+3761 116748506
+3762 54446528
+3763 143690555
+3764 83618046
+3765 146427155
+3766 113319840
+3767 161963596
+3768 18477817
+3769 147449384
+3770 62683945
+3771 167252968
+3772 102779223
+3773 161514717
+3774 132380572
+3775 172394265
+3776 10956812
+3777 101102074
+3778 43002053
+3779 130205756
+3780 71299226
+3781 133920023
+3782 99038459
+3783 151777016
+3784 12088141
+3785 132592265
+3786 47608058
+3787 158079280
+3788 88290584
+3789 151867427
+3790 116615227
+3791 162490543
+3792 20850605
+3793 118248660
+3794 62870898
+3795 149571216
+3796 89090723
+3797 148092040
+3798 114054838
+3799 163913391
+3800 23445531
+3801 149794142
+3802 70882460
+3803 171875921
+3804 107900818
+3805 163115546
+3806 135721673
+3807 180067125
+3808 19699492
+3809 108070773
+3810 59910687
+3811 135935221
+3812 80568288
+3813 141276298
+3814 108819382
+3815 157845656
+3816 20514116
+3817 137746409
+3818 63866545
+3819 161378691
+3820 97766135
+3821 157558868
+3822 127330397
+3823 171548999
+3824 33555620
+3825 124700332
+3826 77630536
+3827 155131475
+3828 98890344
+3829 155391062
+3830 126564270
+3831 170709292
+3832 35510149
+3833 159262817
+3834 86808777
+3835 179591788
+3836 119007141
+3837 171607763
+3838 145185651
+3839 179584537
+3840 5248834
+3841 94381395
+3842 25142684
+3843 125029977
+3844 62374088
+3845 128849043
+3846 87871804
+3847 144188233
+3848 6665412
+3849 127671610
+3850 29784285
+3851 149529628
+3852 79920105
+3853 144288880
+3854 106776349
+3855 157779438
+3856 10815298
+3857 113220237
+3858 40445543
+3859 140762877
+3860 75773653
+3861 140632143
+3862 103623603
+3863 157234398
+3864 13664172
+3865 141496529
+3866 49047139
+3867 164327348
+3868 96423815
+3869 156918208
+3870 123983494
+3871 169639736
+3872 9955276
+3873 98929189
+3874 38782591
+3875 129402611
+3876 69690028
+3877 132179393
+3878 96671076
+3879 149545854
+3880 11299540
+3881 131048249
+3882 43683026
+3883 153593878
+3884 86980793
+3885 150579225
+3886 113721479
+3887 163433378
+3888 18288118
+3889 117626580
+3890 57268114
+3891 147686268
+3892 85663912
+3893 149237379
+3894 114987586
+3895 161312153
+3896 21048857
+3897 148158173
+3898 65424235
+3899 169280130
+3900 103771906
+3901 161537030
+3902 134288168
+3903 173792979
+3904 12960511
+3905 102519133
+3906 45369427
+3907 130691609
+3908 74068241
+3909 134680537
+3910 100765318
+3911 154124084
+3912 14381256
+3913 132503803
+3914 50468497
+3915 155422899
+3916 89987601
+3917 153357094
+3918 119133919
+3919 164333818
+3920 23082249
+3921 119365524
+3922 64705274
+3923 151757145
+3924 90824050
+3925 152245676
+3926 115777440
+3927 163515343
+3928 26156712
+3929 150905780
+3930 73560531
+3931 172536320
+3932 108057388
+3933 165768765
+3934 141263878
+3935 176260371
+3936 21942088
+3937 107555201
+3938 62268749
+3939 138199607
+3940 82465961
+3941 140193308
+3942 110310594
+3943 157889133
+3944 22933325
+3945 142090051
+3946 66132058
+3947 161192310
+3948 99645335
+3949 157021730
+3950 128658914
+3951 171411723
+3952 35270945
+3953 127187831
+3954 80057312
+3955 156845398
+3956 101331709
+3957 156490842
+3958 128083673
+3959 170342057
+3960 38086484
+3961 160501835
+3962 89844274
+3963 177897904
+3964 120122911
+3965 170382486
+3966 148410215
+3967 184966002
+3968 5237981
+3969 93411871
+3970 25319643
+3971 123864599
+3972 62432373
+3973 126750787
+3974 87295995
+3975 143712293
+3976 6594523
+3977 123879692
+3978 29920319
+3979 149917197
+3980 78206479
+3981 147166497
+3982 106047644
+3983 157156928
+3984 10859551
+3985 111691920
+3986 40631403
+3987 140513806
+3988 76634426
+3989 139063133
+3990 103771092
+3991 153582149
+3992 13641885
+3993 143790093
+3994 49695303
+3995 163514194
+3996 96529521
+3997 155556407
+3998 123318955
+3999 169289615
+4000 10112943
+4001 98439518
+4002 38567208
+4003 129333738
+4004 69035501
+4005 133651770
+4006 96969489
+4007 150635675
+4008 11249013
+4009 131233435
+4010 43363397
+4011 156228080
+4012 86040917
+4013 151548517
+4014 114912316
+4015 160874557
+4016 18240181
+4017 118555607
+4018 56463158
+4019 147103894
+4020 85801277
+4021 146905477
+4022 114321362
+4023 162251141
+4024 21145968
+4025 146676116
+4026 65586396
+4027 169390668
+4028 102946385
+4029 160439373
+4030 132905650
+4031 173373924
+4032 12954347
+4033 102563760
+4034 45569611
+4035 131327590
+4036 72940525
+4037 134777311
+4038 101461656
+4039 152372568
+4040 14264745
+4041 132941325
+4042 50226554
+4043 156912202
+4044 90498278
+4045 153052604
+4046 118498746
+4047 166768968
+4048 23284733
+4049 120538340
+4050 64549010
+4051 150848894
+4052 90680852
+4053 147981786
+4054 117662854
+4055 164602162
+4056 26085488
+4057 152423406
+4058 73920499
+4059 170538392
+4060 109528224
+4061 167335477
+4062 137072713
+4063 175327679
+4064 21847541
+4065 108696160
+4066 61227549
+4067 136930815
+4068 83359820
+4069 141490316
+4070 110275654
+4071 158333630
+4072 23189254
+4073 139009910
+4074 64714534
+4075 164912506
+4076 101215697
+4077 157685236
+4078 128683417
+4079 171643294
+4080 35611916
+4081 129566457
+4082 79616243
+4083 153844045
+4084 101873461
+4085 157591516
+4086 127786211
+4087 172202256
+4088 38748208
+4089 159566906
+4090 87797284
+4091 176448362
+4092 120944029
+4093 171350081
+4094 146605383
+4095 183301627
[08/13] kylin git commit: KYLIN-2718 bug fix
Posted by li...@apache.org.
KYLIN-2718 bug fix
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2edee491
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2edee491
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2edee491
Branch: refs/heads/pr70
Commit: 2edee491577d1507d3e251fcb33b5f43c21749d3
Parents: 2551d48
Author: Li Yang <li...@apache.org>
Authored: Sat Sep 23 16:55:07 2017 +0800
Committer: Li Yang <li...@apache.org>
Committed: Sat Sep 23 16:55:07 2017 +0800
----------------------------------------------------------------------
core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java | 2 ++
1 file changed, 2 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/2edee491/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
index 712af43..827ac81 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
@@ -660,6 +660,8 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware {
+ ". Use 'mandatory'/'hierarchy'/'joint' to optimize; or update 'kylin.cube.aggrgroup.max-combination' to a bigger value.";
throw new TooManyCuboidException(msg);
}
+ } catch (TooManyCuboidException e) {
+ throw e;
} catch (Exception e) {
throw new IllegalStateException("Unknown error while calculating cuboid number for " + //
"Aggregation group " + index + " of Cube Desc " + this.name, e);
[10/13] kylin git commit: KYLIN-2802 Enable cube planner phase one
Posted by li...@apache.org.
KYLIN-2802 Enable cube planner phase one
Signed-off-by: Li Yang <li...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f0bc691e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f0bc691e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f0bc691e
Branch: refs/heads/pr70
Commit: f0bc691ecff2a8f92a3a43e49c5c7b1b676d6d10
Parents: 7b6606a
Author: Zhong <nj...@apache.org>
Authored: Fri Sep 15 13:27:27 2017 +0800
Committer: Li Yang <li...@apache.org>
Committed: Sat Sep 23 18:16:48 2017 +0800
----------------------------------------------------------------------
.../apache/kylin/common/KylinConfigBase.java | 27 +++
.../org/apache/kylin/common/util/JsonUtil.java | 3 +-
.../org/apache/kylin/cube/CubeInstance.java | 39 ++++-
.../java/org/apache/kylin/cube/CubeManager.java | 4 +
.../java/org/apache/kylin/cube/CubeUpdate.java | 11 ++
.../cuboid/algorithm/CuboidRecommender.java | 163 +++++++++++++++++++
.../cube/cuboid/algorithm/CuboidStats.java | 27 ++-
.../cube/cuboid/algorithm/CuboidStatsUtil.java | 134 +++++++++------
.../org/apache/kylin/cube/CubeInstanceTest.java | 57 +++++++
.../cuboid/algorithm/CuboidStatsUtilTest.java | 45 +++--
.../src/test/resources/learn_kylin_cube2.json | 84 ++++++++++
.../kylin/engine/mr/common/CubeStatsReader.java | 4 +
.../engine/mr/common/CuboidRecommenderUtil.java | 106 ++++++++++++
.../engine/mr/common/CuboidStatsReaderUtil.java | 135 +++++++++++++++
.../mr/common/StatisticsDecisionUtil.java | 110 +++++++++++++
.../engine/mr/steps/SaveStatisticsStep.java | 55 +------
16 files changed, 880 insertions(+), 124 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/f0bc691e/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index d535b7d..eb2adab 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -1224,6 +1224,33 @@ abstract public class KylinConfigBase implements Serializable {
}
// ============================================================================
+ // Cube Planner
+ // ============================================================================
+ public boolean isCubePlannerEnabledForExistingCube() {
+ return Boolean.parseBoolean(getOptional("kylin.cube.cubeplanner.enabled-for-existing-cube", "false"));
+ }
+
+ public double getCubePlannerExpansionRateThreshold() {
+ return Double.parseDouble(getOptional("kylin.cube.cubeplanner.expansion-threshold", "15.0"));
+ }
+
+ public int getCubePlannerRecommendCuboidCacheMaxSize() {
+ return Integer.parseInt(getOptional("kylin.cube.cubeplanner.recommend-cache-max-size", "200"));
+ }
+
+ public long getCubePlannerMandatoryRollUpThreshold() {
+ return Long.parseLong(getOptional("kylin.cube.cubeplanner.mandatory-rollup-threshold", "1000"));
+ }
+
+ public int getCubePlannerAgreedyAlgorithmAutoThreshold() {
+ return Integer.parseInt(getOptional("kylin.cube.cubeplanner.algorithm-threshold-greedy", "10"));
+ }
+
+ public int getCubePlannerGeneticAlgorithmAutoThreshold() {
+ return Integer.parseInt(getOptional("kylin.cube.cubeplanner.algorithm-threshold-genetic", "23"));
+ }
+
+ // ============================================================================
// RESTCLIENT
// ============================================================================
http://git-wip-us.apache.org/repos/asf/kylin/blob/f0bc691e/core-common/src/main/java/org/apache/kylin/common/util/JsonUtil.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/JsonUtil.java b/core-common/src/main/java/org/apache/kylin/common/util/JsonUtil.java
index 5a81463..01fd134 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/JsonUtil.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/JsonUtil.java
@@ -72,7 +72,8 @@ public class JsonUtil {
return mapper.readValue(src, valueType);
}
- public static <T> T readValue(String content, TypeReference<T> valueTypeRef) throws IOException {
+ public static <T> T readValue(String content, TypeReference<T> valueTypeRef)
+ throws IOException, JsonParseException, JsonMappingException {
return mapper.readValue(content, valueTypeRef);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f0bc691e/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
index 7d539c6..a611b96 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
@@ -19,7 +19,6 @@
package org.apache.kylin.cube;
import java.io.IOException;
-import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -30,6 +29,7 @@ import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.persistence.RootPersistentEntity;
import org.apache.kylin.common.util.CompressionUtils;
import org.apache.kylin.common.util.JsonUtil;
+import org.apache.kylin.cube.cuboid.CuboidModeEnum;
import org.apache.kylin.cube.cuboid.CuboidScheduler;
import org.apache.kylin.cube.cuboid.TreeCuboidScheduler;
import org.apache.kylin.cube.model.CubeDesc;
@@ -328,7 +328,36 @@ public class CubeInstance extends RootPersistentEntity implements IRealization,
this.createTimeUTC = createTimeUTC;
}
- Map<Long, Long> getCuboids() {
+ public Set<Long> getCuboidsByMode(String cuboidModeName) {
+ return getCuboidsByMode(CuboidModeEnum.getByModeName(cuboidModeName));
+ }
+
+ public Set<Long> getCuboidsByMode(CuboidModeEnum cuboidMode) {
+ if (cuboidMode == null || cuboidMode == CuboidModeEnum.CURRENT) {
+ return getCuboidScheduler().getAllCuboidIds();
+ }
+ Set<Long> cuboidsRecommend = getCuboidsRecommend();
+ if (cuboidsRecommend == null || cuboidMode == CuboidModeEnum.RECOMMEND) {
+ return cuboidsRecommend;
+ }
+ Set<Long> currentCuboids = getCuboidScheduler().getAllCuboidIds();
+ switch (cuboidMode) {
+ case RECOMMEND_EXISTING:
+ cuboidsRecommend.retainAll(currentCuboids);
+ return cuboidsRecommend;
+ case RECOMMEND_MISSING:
+ cuboidsRecommend.removeAll(currentCuboids);
+ return cuboidsRecommend;
+ case RECOMMEND_MISSING_WITH_BASE:
+ cuboidsRecommend.removeAll(currentCuboids);
+ currentCuboids.add(getCuboidScheduler().getBaseCuboidId());
+ return cuboidsRecommend;
+ default:
+ return null;
+ }
+ }
+
+ public Map<Long, Long> getCuboids() {
if (cuboidBytes == null)
return null;
byte[] uncompressed;
@@ -344,7 +373,7 @@ public class CubeInstance extends RootPersistentEntity implements IRealization,
}
}
- void setCuboids(Map<Long, Long> cuboids) {
+ public void setCuboids(Map<Long, Long> cuboids) {
if (cuboids == null)
return;
if (cuboids.isEmpty()) {
@@ -361,7 +390,7 @@ public class CubeInstance extends RootPersistentEntity implements IRealization,
}
}
- Set<Long> getCuboidsRecommend() {
+ public Set<Long> getCuboidsRecommend() {
if (cuboidBytesRecommend == null)
return null;
byte[] uncompressed;
@@ -377,7 +406,7 @@ public class CubeInstance extends RootPersistentEntity implements IRealization,
}
}
- void setCuboidsRecommend(HashSet<Long> cuboids) {
+ public void setCuboidsRecommend(Set<Long> cuboids) {
if (cuboids == null)
return;
if (cuboids.isEmpty()) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/f0bc691e/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index 425e32f..d4d88fd 100755
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -418,6 +418,10 @@ public class CubeManager implements IRealizationProvider {
cube.setCost(update.getCost());
}
+ if(update.getCuboids() != null){
+ cube.setCuboids(update.getCuboids());
+ }
+
try {
getStore().putResource(cube.getResourcePath(), cube, CUBE_SERIALIZER);
} catch (IllegalStateException ise) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/f0bc691e/core-cube/src/main/java/org/apache/kylin/cube/CubeUpdate.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeUpdate.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeUpdate.java
index dffaa48..fae20dc 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeUpdate.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeUpdate.java
@@ -18,6 +18,8 @@
package org.apache.kylin.cube;
+import java.util.Map;
+
import org.apache.kylin.metadata.realization.RealizationStatusEnum;
/**
@@ -31,6 +33,7 @@ public class CubeUpdate {
private RealizationStatusEnum status;
private String owner;
private int cost = -1;
+ private Map<Long, Long> cuboids = null;
public CubeUpdate(CubeInstance cubeInstance) {
this.cubeInstance = cubeInstance;
@@ -98,4 +101,12 @@ public class CubeUpdate {
this.cost = cost;
return this;
}
+
+ public Map<Long, Long> getCuboids() {
+ return cuboids;
+ }
+
+ public void setCuboids(Map<Long, Long> cuboids) {
+ this.cuboids = cuboids;
+ }
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f0bc691e/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/CuboidRecommender.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/CuboidRecommender.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/CuboidRecommender.java
new file mode 100644
index 0000000..43b2318
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/CuboidRecommender.java
@@ -0,0 +1,163 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.cube.cuboid.algorithm;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.cube.cuboid.algorithm.generic.GeneticAlgorithm;
+import org.apache.kylin.cube.cuboid.algorithm.greedy.GreedyAlgorithm;
+import org.apache.kylin.metadata.cachesync.Broadcaster;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.cache.Cache;
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.RemovalListener;
+import com.google.common.cache.RemovalNotification;
+import com.google.common.collect.Maps;
+
+public class CuboidRecommender {
+ private static final Logger logger = LoggerFactory.getLogger(CuboidRecommender.class);
+
+ private static Cache<String, Map<Long, Long>> cuboidRecommendCache = CacheBuilder.newBuilder()
+ .removalListener(new RemovalListener<String, Map<Long, Long>>() {
+ @Override
+ public void onRemoval(RemovalNotification<String, Map<Long, Long>> notification) {
+ logger.info("Dict with resource path " + notification.getKey() + " is removed due to "
+ + notification.getCause());
+ }
+ }).maximumSize(KylinConfig.getInstanceFromEnv().getCubePlannerRecommendCuboidCacheMaxSize())
+ .expireAfterWrite(1, TimeUnit.DAYS).build();
+
+ private class CuboidRecommenderSyncListener extends Broadcaster.Listener {
+ @Override
+ public void onClearAll(Broadcaster broadcaster) throws IOException {
+ cuboidRecommendCache.invalidateAll();
+ }
+
+ @Override
+ public void onEntityChange(Broadcaster broadcaster, String entity, Broadcaster.Event event, String cacheKey)
+ throws IOException {
+ cuboidRecommendCache.invalidate(cacheKey);
+ }
+ }
+
+ public CuboidRecommender() {
+ Broadcaster.getInstance(KylinConfig.getInstanceFromEnv()).registerListener(new CuboidRecommenderSyncListener(),
+ "cube");
+ }
+
+ private static CuboidRecommender instance = new CuboidRecommender();
+
+ public static CuboidRecommender getInstance() {
+ return instance;
+ }
+
+ public Map<Long, Long> getRecommendCuboidList(final CuboidStats cuboidStats, final KylinConfig kylinConfig) {
+ if (cuboidStats == null) {
+ return null;
+ }
+ final String key = cuboidStats.getKey();
+ Map<Long, Long> results = cuboidRecommendCache.getIfPresent(key);
+ if (results == null) {
+ try {
+ results = cuboidRecommendCache.get(key, new Callable<Map<Long, Long>>() {
+ @Override
+ public Map<Long, Long> call() throws Exception {
+ // recommending flag
+ Map<Long, Long> emptyMap = Maps.newHashMap();
+ cuboidRecommendCache.put(key, emptyMap);
+ try {
+ Map<Long, Long> recommendCuboid = getRecommendCuboidList(cuboidStats, kylinConfig,
+ true);
+
+ if (recommendCuboid != null) {
+ logger.info("Add recommend cuboids for " + key + " to cache");
+ cuboidRecommendCache.put(key, recommendCuboid);
+ }
+
+ return recommendCuboid;
+ } catch (Exception e) {
+ cuboidRecommendCache.invalidate(key);
+ logger.error("Failed to get recommend cuboids for " + key + " in cache", e);
+ throw e;
+ }
+ }
+ });
+ } catch (ExecutionException e) {
+ logger.error("Failed to get recommend cuboids for " + key);
+ }
+ }
+ return results;
+ }
+
+ public Map<Long, Long> getRecommendCuboidList(CuboidStats cuboidStats, KylinConfig kylinConf,
+ boolean ifForceRecommend) {
+ long Threshold1 = 1L << kylinConf.getCubePlannerAgreedyAlgorithmAutoThreshold();
+ long Threshold2 = 1L << kylinConf.getCubePlannerGeneticAlgorithmAutoThreshold();
+ if (Threshold1 >= Threshold2) {
+ logger.error("Invalid Cube Planner Algorithm configuration");
+ return null;
+ }
+
+ int allCuboidCount = cuboidStats.getAllCuboidsForMandatory().size()
+ + cuboidStats.getAllCuboidsForSelection().size();
+
+ BenefitPolicy benefitPolicy = new PBPUSCalculator(cuboidStats);
+ CuboidRecommendAlgorithm algorithm = null;
+
+ if (allCuboidCount <= Threshold2) {
+ algorithm = new GreedyAlgorithm(-1, benefitPolicy, cuboidStats);
+ } else {
+ algorithm = new GeneticAlgorithm(-1, benefitPolicy, cuboidStats);
+ }
+
+ long startTime = System.currentTimeMillis();
+ logger.info("Cube Planner Algorithm started at " + startTime);
+ List<Long> recommendCuboidList = algorithm.recommend(kylinConf.getCubePlannerExpansionRateThreshold());
+ logger.info("Cube Planner Algorithm ended at " + (System.currentTimeMillis() - startTime));
+
+ if (recommendCuboidList.size() < allCuboidCount) {
+ logger.info("Cube Planner Algorithm chooses " + recommendCuboidList.size()
+ + " most effective cuboids to build among of all " + allCuboidCount + " cuboids.");
+ }
+
+ Map<Long, Long> recommendCuboidsWithStats = Maps.newLinkedHashMap();
+ for (Long cuboid : recommendCuboidList) {
+ if (cuboid.equals(cuboidStats.getBaseCuboid())) {
+ recommendCuboidsWithStats.put(cuboid, cuboidStats.getCuboidCount(cuboid));
+ } else if (cuboidStats.getAllCuboidsForSelection().contains(cuboid)) {
+ recommendCuboidsWithStats.put(cuboid, cuboidStats.getCuboidCount(cuboid));
+ } else {
+ recommendCuboidsWithStats.put(cuboid, -1L);
+ }
+ }
+
+ if (!ifForceRecommend && allCuboidCount <= Threshold1) {
+ return null;
+ }
+ return recommendCuboidsWithStats;
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f0bc691e/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/CuboidStats.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/CuboidStats.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/CuboidStats.java
index 1775d5a..a1c191e 100755
--- a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/CuboidStats.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/CuboidStats.java
@@ -18,6 +18,7 @@
package org.apache.kylin.cube.cuboid.algorithm;
+import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -120,7 +121,8 @@ public class CuboidStats {
private ImmutableMap<Long, Double> cuboidHitProbabilityMap;
private ImmutableMap<Long, Long> cuboidScanCountMap;
- private ImmutableMap<Long, Set<Long>> allDescendantsCache;
+ private ImmutableMap<Long, List<Long>> directChildrenCache;
+ private Map<Long, Set<Long>> allDescendantsCache;
private CuboidStats(String key, long baseCuboidId, Set<Long> mandatoryCuboids, Map<Long, Long> statistics,
Map<Long, Double> size, Map<Long, Long> hitFrequencyMap, Map<Long, Map<Long, Long>> scanCountSourceMap) {
@@ -187,8 +189,10 @@ public class CuboidStats {
}
this.cuboidScanCountMap = ImmutableMap.<Long, Long> builder().putAll(tmpCuboidScanCountMap).build();
- this.allDescendantsCache = ImmutableMap.<Long, Set<Long>> builder()
- .putAll(CuboidStatsUtil.createAllDescendantsCache(statistics.keySet())).build();
+ this.directChildrenCache = ImmutableMap.<Long, List<Long>> builder()
+ .putAll(CuboidStatsUtil.createDirectChildrenCache(statistics.keySet())).build();
+
+ this.allDescendantsCache = Maps.newConcurrentMap();
}
private long getExpScanCount(long sourceCuboid, Map<Long, Long> statistics,
@@ -216,11 +220,26 @@ public class CuboidStats {
public Set<Long> getAllDescendants(long cuboid) {
Set<Long> allDescendants = Sets.newLinkedHashSet();
if (selectionCuboidSet.contains(cuboid)) {
- return allDescendantsCache.get(cuboid);
+ if (allDescendantsCache.get(cuboid) != null) {
+ return allDescendantsCache.get(cuboid);
+ } else {
+ getAllDescendants(cuboid, allDescendants);
+ allDescendantsCache.put(cuboid, allDescendants);
+ }
}
return allDescendants;
}
+ private void getAllDescendants(long cuboid, Set<Long> allDescendants) {
+ if (allDescendants.contains(cuboid)) {
+ return;
+ }
+ allDescendants.add(cuboid);
+ for (Long directChild : directChildrenCache.get(cuboid)) {
+ getAllDescendants(directChild, allDescendants);
+ }
+ }
+
public Set<Long> getAllCuboidsForSelection() {
return selectionCuboidSet;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f0bc691e/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/CuboidStatsUtil.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/CuboidStatsUtil.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/CuboidStatsUtil.java
index 6d5bbe5..c2683df 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/CuboidStatsUtil.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/CuboidStatsUtil.java
@@ -20,13 +20,13 @@ package org.apache.kylin.cube.cuboid.algorithm;
import java.util.Collections;
import java.util.Comparator;
+import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.SortedSet;
import java.util.TreeSet;
-import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
@@ -102,62 +102,102 @@ public class CuboidStatsUtil {
}
}
- /** Using dynamic programming to use extra space to reduce repetitive computation*/
- public static Map<Long, Set<Long>> createAllDescendantsCache(final Set<Long> cuboidSet) {
- List<Long> latticeCuboidList = Lists.newArrayList(cuboidSet);
- Collections.sort(latticeCuboidList);
-
- Map<Long, Set<Long>> allDescendantsCache = Maps.newHashMap();
- Set<Long> preNoneDescendants = Sets.newHashSet();
- for (int i = 0; i < latticeCuboidList.size(); i++) {
- Long currentCuboid = latticeCuboidList.get(i);
- Set<Long> descendants = Sets.newHashSet(currentCuboid);
- Set<Long> curNoneDescendants = Sets.newHashSet();
- if (i > 0) {
- long preCuboid = latticeCuboidList.get(i - 1);
- if (isDescendant(preCuboid, currentCuboid)) {
- descendants.addAll(allDescendantsCache.get(preCuboid));
- } else {
- curNoneDescendants.add(preCuboid);
- for (long cuboidToCheck : allDescendantsCache.get(preCuboid)) {
- if (isDescendant(cuboidToCheck, currentCuboid)) {
- descendants.addAll(allDescendantsCache.get(cuboidToCheck));
- }
- }
- }
- }
- for (long cuboidToCheck : preNoneDescendants) {
- if (isDescendant(cuboidToCheck, currentCuboid)) {
- descendants.addAll(allDescendantsCache.get(cuboidToCheck));
- } else {
- curNoneDescendants.add(cuboidToCheck);
+ public static Map<Long, List<Long>> createDirectChildrenCache(final Set<Long> cuboidSet) {
+ /**
+ * Sort the list by ascending order:
+ * */
+ final List<Long> cuboidList = Lists.newArrayList(cuboidSet);
+ Collections.sort(cuboidList);
+ /**
+ * Sort the list by ascending order:
+ * 1. the more bit count of its value, the bigger
+ * 2. the larger of its value, the bigger
+ * */
+ List<Integer> layerIdxList = Lists.newArrayListWithExpectedSize(cuboidList.size());
+ for (int i = 0; i < cuboidList.size(); i++) {
+ layerIdxList.add(i);
+ }
+ Collections.sort(layerIdxList, new Comparator<Integer>() {
+ @Override
+ public int compare(Integer i1, Integer i2) {
+ Long o1 = cuboidList.get(i1);
+ Long o2 = cuboidList.get(i2);
+ int nBitDiff = Long.bitCount(o1) - Long.bitCount(o2);
+ if (nBitDiff != 0) {
+ return nBitDiff;
}
+ return Long.compare(o1, o2);
}
+ });
+ /**
+ * Construct an index array for pointing the position in layerIdxList
+ * (layerCuboidList is for speeding up continuous iteration)
+ * */
+ int[] toLayerIdxArray = new int[layerIdxList.size()];
+ final List<Long> layerCuboidList = Lists.newArrayListWithExpectedSize(cuboidList.size());
+ for (int i = 0; i < layerIdxList.size(); i++) {
+ int cuboidIdx = layerIdxList.get(i);
+ toLayerIdxArray[cuboidIdx] = i;
+ layerCuboidList.add(cuboidList.get(cuboidIdx));
+ }
- allDescendantsCache.put(currentCuboid, descendants);
- preNoneDescendants = curNoneDescendants;
+ int[] previousLayerLastIdxArray = new int[layerIdxList.size()];
+ int currentBitCount = 0;
+ int previousLayerLastIdx = -1;
+ for (int i = 0; i < layerIdxList.size(); i++) {
+ int cuboidIdx = layerIdxList.get(i);
+ int nBits = Long.bitCount(cuboidList.get(cuboidIdx));
+ if (nBits > currentBitCount) {
+ currentBitCount = nBits;
+ previousLayerLastIdx = i - 1;
+ }
+ previousLayerLastIdxArray[i] = previousLayerLastIdx;
}
- return allDescendantsCache;
+ Map<Long, List<Long>> directChildrenCache = Maps.newHashMap();
+ for (int i = 0; i < cuboidList.size(); i++) {
+ Long currentCuboid = cuboidList.get(i);
+ LinkedList<Long> directChildren = Lists.newLinkedList();
+ int lastLayerIdx = previousLayerLastIdxArray[toLayerIdxArray[i]];
+ /**
+ * Choose one of the two scan strategies
+ * 1. cuboids are sorted by its value, like 1,2,3,4,...
+ * 2. cuboids are layered and sorted, like 1,2,4,8,...,3,5,...
+ * */
+ if (i - 1 <= lastLayerIdx) {
+ /**
+ * 1. Adding cuboid by descending order
+ * */
+ for (int j = i - 1; j >= 0; j--) {
+ checkAndAddDirectChild(directChildren, currentCuboid, cuboidList.get(j));
+ }
+ } else {
+ /**
+ * 1. Adding cuboid by descending order
+ * 2. Check from lower cuboid layer
+ * */
+ for (int j = lastLayerIdx; j >= 0; j--) {
+ checkAndAddDirectChild(directChildren, currentCuboid, layerCuboidList.get(j));
+ }
+ }
+ directChildrenCache.put(currentCuboid, directChildren);
+ }
+ return directChildrenCache;
}
- @VisibleForTesting
- static Map<Long, Set<Long>> createAllDescendantsCache2(final Set<Long> cuboidSet) {
- List<Long> latticeCuboidList = Lists.newArrayList(cuboidSet);
-
- Map<Long, Set<Long>> allDescendantsCache = Maps.newHashMap();
- for (int i = 0; i < latticeCuboidList.size(); i++) {
- Long currentCuboid = latticeCuboidList.get(i);
- Set<Long> descendantSet = Sets.newHashSet(currentCuboid);
- for (int j = 0; j < i; j++) {
- Long checkCuboid = latticeCuboidList.get(j);
- if (isDescendant(checkCuboid, currentCuboid)) {
- descendantSet.add(checkCuboid);
+ private static void checkAndAddDirectChild(List<Long> directChildren, Long currentCuboid, Long checkedCuboid) {
+ if (isDescendant(checkedCuboid, currentCuboid)) {
+ boolean ifDirectChild = true;
+ for (long directChild : directChildren) {
+ if (isDescendant(checkedCuboid, directChild)) {
+ ifDirectChild = false;
+ break;
}
}
- allDescendantsCache.put(currentCuboid, descendantSet);
+ if (ifDirectChild) {
+ directChildren.add(checkedCuboid);
+ }
}
- return allDescendantsCache;
}
public static boolean isDescendant(long cuboidToCheck, long parentCuboid) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/f0bc691e/core-cube/src/test/java/org/apache/kylin/cube/CubeInstanceTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/CubeInstanceTest.java b/core-cube/src/test/java/org/apache/kylin/cube/CubeInstanceTest.java
new file mode 100644
index 0000000..0af5f98
--- /dev/null
+++ b/core-cube/src/test/java/org/apache/kylin/cube/CubeInstanceTest.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.cube;
+
+import java.io.DataInputStream;
+import java.io.FileInputStream;
+import java.io.InputStream;
+import java.util.Map;
+
+import org.apache.kylin.common.persistence.JsonSerializer;
+import org.apache.kylin.cube.cuboid.TreeCuboidScheduler;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.google.common.collect.Lists;
+
+public class CubeInstanceTest {
+
+ private CubeInstance cubeInstance;
+
+ @Before
+ public void setUp() throws Exception {
+ InputStream fileInputStream = new FileInputStream("src/test/resources/learn_kylin_cube2.json");
+ JsonSerializer<CubeInstance> jsonSerializer = new JsonSerializer<>(CubeInstance.class);
+ cubeInstance = jsonSerializer.deserialize(new DataInputStream(fileInputStream));
+ }
+
+ @Test
+ public void getTreeCuboidSchedulerTest() {
+ Map<Long, Long> cuboidsWithRowCnt = cubeInstance.getCuboids();
+ TreeCuboidScheduler.CuboidTree.createFromCuboids(Lists.newArrayList(cuboidsWithRowCnt.keySet()),
+ new TreeCuboidScheduler.CuboidCostComparator(cuboidsWithRowCnt));
+ }
+
+ public void printMap(Map<Long, Long> map) {
+ System.out.println("size: " + map.size());
+ for (Map.Entry<Long, Long> entry : map.entrySet()) {
+ System.out.println(entry.getKey() + ":" + entry.getValue());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f0bc691e/core-cube/src/test/java/org/apache/kylin/cube/cuboid/algorithm/CuboidStatsUtilTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/cuboid/algorithm/CuboidStatsUtilTest.java b/core-cube/src/test/java/org/apache/kylin/cube/cuboid/algorithm/CuboidStatsUtilTest.java
index ca70820..cb2b699 100644
--- a/core-cube/src/test/java/org/apache/kylin/cube/cuboid/algorithm/CuboidStatsUtilTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/cube/cuboid/algorithm/CuboidStatsUtilTest.java
@@ -19,14 +19,15 @@
package org.apache.kylin.cube.cuboid.algorithm;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
import java.util.Set;
-import java.util.concurrent.TimeUnit;
import org.junit.Assert;
import org.junit.Test;
import com.google.common.base.Stopwatch;
+import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
@@ -122,16 +123,16 @@ public class CuboidStatsUtilTest {
}
@Test
- public void createAllDescendantsCacheTest() {
+ public void createDirectChildrenCacheTest() {
Set<Long> cuboidSet = generateCuboidSet();
- Map<Long, Set<Long>> allDescendantsCache = CuboidStatsUtil.createAllDescendantsCache(cuboidSet);
-
- Assert.assertTrue(allDescendantsCache.get(255L).containsAll(cuboidSet));
-
- Assert.assertTrue(allDescendantsCache.get(239L).size() == 5);
-
- Assert.assertTrue(allDescendantsCache.get(50L).containsAll(Sets.newHashSet(50L, 2L)));
- Assert.assertTrue(!allDescendantsCache.get(50L).contains(4L));
+ Map<Long, List<Long>> directChildrenCache = CuboidStatsUtil.createDirectChildrenCache(cuboidSet);
+
+ Assert.assertTrue(directChildrenCache.get(255L).containsAll(Lists.newArrayList(239L, 159L, 50L)));
+ Assert.assertTrue(directChildrenCache.get(159L).contains(6L));
+ Assert.assertTrue(directChildrenCache.get(50L).contains(2L));
+ Assert.assertTrue(directChildrenCache.get(239L).contains(199L));
+ Assert.assertTrue(directChildrenCache.get(199L).contains(6L));
+ Assert.assertTrue(directChildrenCache.get(6L).containsAll(Lists.newArrayList(4L, 2L)));
}
private Set<Long> generateMassCuboidSet() {
@@ -144,18 +145,26 @@ public class CuboidStatsUtilTest {
}
@Test
- public void createAllDescendantsCacheStressTest() {
+ public void createDirectChildrenCacheStressTest() {
Stopwatch sw = new Stopwatch();
sw.start();
Set<Long> cuboidSet = generateMassCuboidSet();
- System.out.println("Time elapsed for creating sorted cuboid list: " + sw.elapsed(TimeUnit.MILLISECONDS));
- sw.reset();
- sw.start();
- CuboidStatsUtil.createAllDescendantsCache(cuboidSet);
- System.out.println("Time elapsed for creating descendants cache: " + sw.elapsed(TimeUnit.MILLISECONDS));
+ System.out.println("Time elapsed for creating sorted cuboid list: " + sw.elapsedMillis());
sw.reset();
sw.start();
- CuboidStatsUtil.createAllDescendantsCache2(cuboidSet);
- System.out.println("Time elapsed for creating descendants cache2: " + sw.elapsed(TimeUnit.MILLISECONDS));
+ checkDirectChildrenCacheStressTest(CuboidStatsUtil.createDirectChildrenCache(cuboidSet));
+ System.out.println("Time elapsed for creating direct children cache: " + sw.elapsedMillis());
+ sw.stop();
+ }
+
+ private void checkDirectChildrenCacheStressTest(Map<Long, List<Long>> directChildrenCache) {
+ for (Map.Entry<Long, List<Long>> entry : directChildrenCache.entrySet()) {
+ if (Long.bitCount(entry.getKey()) == 1) {
+ Assert.assertTrue("Check for cuboid " + entry.getKey(), entry.getValue().size() == 0);
+ } else {
+ Assert.assertTrue("Check for cuboid " + entry.getKey(),
+ Long.bitCount(entry.getKey()) == entry.getValue().size());
+ }
+ }
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f0bc691e/core-cube/src/test/resources/learn_kylin_cube2.json
----------------------------------------------------------------------
diff --git a/core-cube/src/test/resources/learn_kylin_cube2.json b/core-cube/src/test/resources/learn_kylin_cube2.json
new file mode 100644
index 0000000..4d731bb
--- /dev/null
+++ b/core-cube/src/test/resources/learn_kylin_cube2.json
@@ -0,0 +1,84 @@
+{
+ "uuid" : "bd66b324-4b0d-4fc7-a2ef-2b9ddced160d",
+ "last_modified" : 1503388415029,
+ "version" : "2.1.0",
+ "name" : "learn_kylin_cube2",
+ "owner" : "ADMIN",
+ "descriptor" : "learn_kylin_cube2",
+ "cost" : 50,
+ "status" : "READY",
+ "segments" : [ {
+ "uuid" : "4d017947-1a63-47a9-b31d-356219612cf6",
+ "name" : "20100101000000_20150101000000",
+ "storage_location_identifier" : "KYLIN_FKF6CWSIZC",
+ "date_range_start" : 1262304000000,
+ "date_range_end" : 1420070400000,
+ "source_offset_start" : 0,
+ "source_offset_end" : 0,
+ "status" : "READY",
+ "size_kb" : 22153,
+ "input_records" : 10000,
+ "input_records_size" : 984072,
+ "last_build_time" : 1503388048037,
+ "last_build_job_id" : "06c5bfd9-0443-4c28-810b-bcbd37fde458",
+ "create_time_utc" : 1503387574466,
+ "cuboid_shard_nums" : { },
+ "total_shards" : 1,
+ "blackout_cuboids" : [ ],
+ "binary_signature" : null,
+ "dictionaries" : {
+ "KYLIN_SALES.LSTG_SITE_ID" : "/dict/DEFAULT.KYLIN_CATEGORY_GROUPINGS/SITE_ID/d1dbea6a-4929-43ef-bf2a-cf356d9de9eb.dict",
+ "KYLIN_SALES.LEAF_CATEG_ID" : "/dict/DEFAULT.KYLIN_CATEGORY_GROUPINGS/LEAF_CATEG_ID/60f6c00a-2137-4d6f-90e6-5433d30e326d.dict",
+ "KYLIN_CATEGORY_GROUPINGS.META_CATEG_NAME" : "/dict/DEFAULT.KYLIN_CATEGORY_GROUPINGS/META_CATEG_NAME/e375c3b2-bebf-4f9a-95f3-abb76a7bdc80.dict",
+ "KYLIN_SALES.SLR_SEGMENT_CD" : "/dict/DEFAULT.KYLIN_SALES/SLR_SEGMENT_CD/ddb1659c-85cc-4d4f-8aae-40c6618f61cc.dict",
+ "KYLIN_CATEGORY_GROUPINGS.CATEG_LVL3_NAME" : "/dict/DEFAULT.KYLIN_CATEGORY_GROUPINGS/CATEG_LVL3_NAME/7bbcdb05-4cf9-4d31-942c-828c5bbcaab8.dict",
+ "KYLIN_SALES.LSTG_FORMAT_NAME" : "/dict/DEFAULT.KYLIN_SALES/LSTG_FORMAT_NAME/35669f98-38f2-4e67-9547-1b0b446a1675.dict",
+ "KYLIN_CATEGORY_GROUPINGS.CATEG_LVL2_NAME" : "/dict/DEFAULT.KYLIN_CATEGORY_GROUPINGS/CATEG_LVL2_NAME/a83a92e5-7711-4a34-bff8-ab3da625e996.dict"
+ },
+ "snapshots" : {
+ "DEFAULT.KYLIN_CATEGORY_GROUPINGS" : "/table_snapshot/DEFAULT.KYLIN_CATEGORY_GROUPINGS/90b8251c-96c9-45a6-a4db-9fe49e8a0581.snapshot",
+ "DEFAULT.KYLIN_CAL_DT" : "/table_snapshot/DEFAULT.KYLIN_CAL_DT/35abb8cf-4e70-428e-adef-457e5a085aa0.snapshot"
+ },
+ "rowkey_stats" : [ [ "KYLIN_SALES.LSTG_FORMAT_NAME", 5, 1 ], [ "KYLIN_SALES.SLR_SEGMENT_CD", 8, 1 ], [ "KYLIN_SALES.LEAF_CATEG_ID", 134, 1 ], [ "KYLIN_SALES.LSTG_SITE_ID", 8, 1 ], [ "KYLIN_CATEGORY_GROUPINGS.META_CATEG_NAME", 44, 1 ], [ "KYLIN_CATEGORY_GROUPINGS.CATEG_LVL2_NAME", 94, 1 ], [ "KYLIN_CATEGORY_GROUPINGS.CATEG_LVL3_NAME", 127, 1 ] ]
+ }, {
+ "uuid" : "4cb8c3e7-9624-44c4-b812-041cb5604e50",
+ "name" : "20100101000000_20150101000000",
+ "storage_location_identifier" : "KYLIN_VEXWSEB5Y7",
+ "date_range_start" : 1262304000000,
+ "date_range_end" : 1420070400000,
+ "source_offset_start" : 0,
+ "source_offset_end" : 0,
+ "status" : "NEW",
+ "size_kb" : 0,
+ "input_records" : 0,
+ "input_records_size" : 0,
+ "last_build_time" : 0,
+ "last_build_job_id" : null,
+ "create_time_utc" : 1503388305176,
+ "cuboid_shard_nums" : { },
+ "total_shards" : 0,
+ "blackout_cuboids" : [ ],
+ "binary_signature" : null,
+ "dictionaries" : {
+ "KYLIN_SALES.LSTG_SITE_ID" : "/dict/DEFAULT.KYLIN_CATEGORY_GROUPINGS/SITE_ID/d1dbea6a-4929-43ef-bf2a-cf356d9de9eb.dict",
+ "KYLIN_SALES.LEAF_CATEG_ID" : "/dict/DEFAULT.KYLIN_CATEGORY_GROUPINGS/LEAF_CATEG_ID/60f6c00a-2137-4d6f-90e6-5433d30e326d.dict",
+ "KYLIN_CATEGORY_GROUPINGS.META_CATEG_NAME" : "/dict/DEFAULT.KYLIN_CATEGORY_GROUPINGS/META_CATEG_NAME/e375c3b2-bebf-4f9a-95f3-abb76a7bdc80.dict",
+ "KYLIN_SALES.SLR_SEGMENT_CD" : "/dict/DEFAULT.KYLIN_SALES/SLR_SEGMENT_CD/ddb1659c-85cc-4d4f-8aae-40c6618f61cc.dict",
+ "KYLIN_CATEGORY_GROUPINGS.CATEG_LVL3_NAME" : "/dict/DEFAULT.KYLIN_CATEGORY_GROUPINGS/CATEG_LVL3_NAME/7bbcdb05-4cf9-4d31-942c-828c5bbcaab8.dict",
+ "KYLIN_SALES.LSTG_FORMAT_NAME" : "/dict/DEFAULT.KYLIN_SALES/LSTG_FORMAT_NAME/35669f98-38f2-4e67-9547-1b0b446a1675.dict",
+ "KYLIN_CATEGORY_GROUPINGS.CATEG_LVL2_NAME" : "/dict/DEFAULT.KYLIN_CATEGORY_GROUPINGS/CATEG_LVL2_NAME/a83a92e5-7711-4a34-bff8-ab3da625e996.dict"
+ },
+ "snapshots" : {
+ "DEFAULT.KYLIN_CATEGORY_GROUPINGS" : "/table_snapshot/DEFAULT.KYLIN_CATEGORY_GROUPINGS/90b8251c-96c9-45a6-a4db-9fe49e8a0581.snapshot",
+ "DEFAULT.KYLIN_CAL_DT" : "/table_snapshot/DEFAULT.KYLIN_CAL_DT/35abb8cf-4e70-428e-adef-457e5a085aa0.snapshot"
+ },
+ "rowkey_stats" : [ [ "KYLIN_SALES.LSTG_FORMAT_NAME", 5, 1 ], [ "KYLIN_SALES.SLR_SEGMENT_CD", 8, 1 ], [ "KYLIN_SALES.LEAF_CATEG_ID", 134, 1 ], [ "KYLIN_SALES.LSTG_SITE_ID", 8, 1 ], [ "KYLIN_CATEGORY_GROUPINGS.META_CATEG_NAME", 44, 1 ], [ "KYLIN_CATEGORY_GROUPINGS.CATEG_LVL2_NAME", 94, 1 ], [ "KYLIN_CATEGORY_GROUPINGS.CATEG_LVL3_NAME", 127, 1 ] ]
+ } ],
+ "create_time_utc" : 1502959568985,
+ "cuboid_bytes" : "eAEdkTmWBTEIA+/iuAM24+Vq8+buv+TINEhCqP9GzDnuOebf6Bp3fmOPu76RMe7m8XG96huL76BRNm4yPz1uGS/0PhSdIG3lN9wQcg09llB9VEo31UyoPg2RMNQqDiQoxdOgwkEJlFrowiCEMzemCDMu8LnRXeBiaisG3yuJ2dm05uG2nG8p454yigDlKsgyYjqOzjPkBizNmSUyW6tc15hWSTgo3MGnp+JAYqd6R8zc2IyX3So5aJC1zgMAPTYpQwEdE9adgDkEqYXZ7hCJhYktJ1t/4glhb9NCEsXzg0kldceG0YtAXb/p5NK4KPdTLsVmkf8/QfNdrA==",
+ "cuboid_bytes_recommend" : null,
+ "last_optimized" : 0,
+ "size_kb" : 22153,
+ "input_records_count" : 10000,
+ "input_records_size" : 984072
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/f0bc691e/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
index 417697d..8f1da6e 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CubeStatsReader.java
@@ -140,6 +140,10 @@ public class CubeStatsReader {
return tempFile;
}
+ public Map<Long, HLLCounter> getCuboidRowEstimatesHLLOrigin() {
+ return this.cuboidRowEstimatesHLL;
+ }
+
public Map<Long, Long> getCuboidRowEstimatesHLL() {
return getCuboidRowCountMapFromSampling(cuboidRowEstimatesHLL, samplingPercentage);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f0bc691e/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CuboidRecommenderUtil.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CuboidRecommenderUtil.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CuboidRecommenderUtil.java
new file mode 100644
index 0000000..ba3f023
--- /dev/null
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CuboidRecommenderUtil.java
@@ -0,0 +1,106 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.engine.mr.common;
+
+import java.io.IOException;
+import java.util.Map;
+
+import org.apache.kylin.common.util.Pair;
+import org.apache.kylin.cube.CubeInstance;
+import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.cube.cuboid.algorithm.CuboidRecommender;
+import org.apache.kylin.cube.cuboid.algorithm.CuboidStats;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class CuboidRecommenderUtil {
+
+ private static final Logger logger = LoggerFactory.getLogger(CuboidRecommenderUtil.class);
+
+ /** Trigger cube planner phase one */
+ public static Map<Long, Long> getRecommendCuboidList(CubeSegment segment) throws IOException {
+ if (segment == null) {
+ return null;
+ }
+
+ CubeStatsReader cubeStatsReader = new CubeStatsReader(segment, segment.getConfig());
+ if (cubeStatsReader.getCuboidRowEstimatesHLL() == null
+ || cubeStatsReader.getCuboidRowEstimatesHLL().isEmpty()) {
+ logger.info("Cuboid Statistics is not enabled.");
+ return null;
+ }
+ long baseCuboid = segment.getCuboidScheduler().getBaseCuboidId();
+ if (cubeStatsReader.getCuboidRowEstimatesHLL().get(baseCuboid) == null
+ || cubeStatsReader.getCuboidRowEstimatesHLL().get(baseCuboid) == 0L) {
+ logger.info("Base cuboid count in cuboid statistics is 0.");
+ return null;
+ }
+
+ String key = segment.getCubeInstance().getName();
+ CuboidStats cuboidStats = new CuboidStats.Builder(key, baseCuboid, cubeStatsReader.getCuboidRowEstimatesHLL(),
+ cubeStatsReader.getCuboidSizeMap()).build();
+ return CuboidRecommender.getInstance().getRecommendCuboidList(cuboidStats, segment.getConfig(), false);
+ }
+
+ /** Trigger cube planner phase two for optimization */
+ public static Map<Long, Long> getRecommendCuboidList(CubeInstance cube, Map<Long, Long> hitFrequencyMap,
+ Map<Long, Map<Long, Long>> rollingUpCountSourceMap) throws IOException {
+
+ Pair<Map<Long, Long>, Map<Long, Double>> statsPair = CuboidStatsReaderUtil
+ .readCuboidStatsAndSizeFromCube(cube.getCuboidScheduler().getAllCuboidIds(), cube);
+
+ String key = cube.getName();
+ long baseCuboid = cube.getCuboidScheduler().getBaseCuboidId();
+ CuboidStats cuboidStats = new CuboidStats.Builder(key, baseCuboid, statsPair.getFirst(), statsPair.getSecond())
+ .setHitFrequencyMap(hitFrequencyMap).setRollingUpCountSourceMap(rollingUpCountSourceMap,
+ cube.getConfig().getCubePlannerMandatoryRollUpThreshold())
+ .build();
+ return CuboidRecommender.getInstance().getRecommendCuboidList(cuboidStats, cube.getConfig());
+ }
+
+ /** For future segment level recommend */
+ public static Map<Long, Long> getRecommendCuboidList(CubeSegment segment, Map<Long, Long> hitFrequencyMap,
+ Map<Long, Map<Long, Long>> rollingUpCountSourceMap, boolean ifForceRecommend) throws IOException {
+ if (segment == null) {
+ return null;
+ }
+
+ CubeStatsReader cubeStatsReader = new CubeStatsReader(segment, segment.getConfig());
+ if (cubeStatsReader.getCuboidRowEstimatesHLL() == null
+ || cubeStatsReader.getCuboidRowEstimatesHLL().isEmpty()) {
+ logger.info("Cuboid Statistics is not enabled.");
+ return null;
+ }
+ long baseCuboid = segment.getCuboidScheduler().getBaseCuboidId();
+ if (cubeStatsReader.getCuboidRowEstimatesHLL().get(baseCuboid) == null
+ || cubeStatsReader.getCuboidRowEstimatesHLL().get(baseCuboid) == 0L) {
+ logger.info("Base cuboid count in cuboid statistics is 0.");
+ return null;
+ }
+
+ String key = segment.getCubeInstance().getName() + "-" + segment.getName();
+ CuboidStats cuboidStats = new CuboidStats.Builder(key, baseCuboid, cubeStatsReader.getCuboidRowEstimatesHLL(),
+ cubeStatsReader.getCuboidSizeMap()).setHitFrequencyMap(hitFrequencyMap)
+ .setRollingUpCountSourceMap(rollingUpCountSourceMap,
+ segment.getConfig().getCubePlannerMandatoryRollUpThreshold())
+ .build();
+ return CuboidRecommender.getInstance().getRecommendCuboidList(cuboidStats, segment.getConfig(),
+ ifForceRecommend);
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f0bc691e/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CuboidStatsReaderUtil.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CuboidStatsReaderUtil.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CuboidStatsReaderUtil.java
new file mode 100644
index 0000000..3025750
--- /dev/null
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/CuboidStatsReaderUtil.java
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.engine.mr.common;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.kylin.common.util.Pair;
+import org.apache.kylin.cube.CubeInstance;
+import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.measure.hllc.HLLCounter;
+import org.apache.kylin.metadata.model.SegmentStatusEnum;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Maps;
+
+public class CuboidStatsReaderUtil {
+
+ private static final Logger logger = LoggerFactory.getLogger(CuboidStatsReaderUtil.class);
+
+ public static Map<Long, Long> readCuboidStatsFromCube(Set<Long> cuboidIds, CubeInstance cubeInstance)
+ throws IOException {
+ Map<Long, Long> statisticsMerged = readCuboidStatsAndSizeFromCube(cuboidIds, cubeInstance).getFirst();
+ return statisticsMerged.isEmpty() ? null : statisticsMerged;
+ }
+
+ public static Pair<Map<Long, Long>, Map<Long, Double>> readCuboidStatsAndSizeFromCube(Set<Long> cuboidIds,
+ CubeInstance cube) throws IOException {
+ Preconditions.checkNotNull(cuboidIds, "The cuboid set can not be null");
+ Preconditions.checkNotNull(cube, "The cube instance can not be null");
+
+ List<CubeSegment> segmentList = cube.getSegments(SegmentStatusEnum.READY);
+ Map<Long, Long> statisticsMerged = Maps.newHashMapWithExpectedSize(cuboidIds.size());
+ Map<Long, Double> sizeMerged = Maps.newHashMapWithExpectedSize(cuboidIds.size());
+ readCuboidStatsFromSegments(cuboidIds, segmentList, statisticsMerged, sizeMerged);
+ return new Pair<>(statisticsMerged, sizeMerged);
+ }
+
+ public static Map<Long, Long> readCuboidStatsFromSegments(Set<Long> cuboidIds, List<CubeSegment> segmentList)
+ throws IOException {
+ Map<Long, Long> statisticsMerged = Maps.newHashMapWithExpectedSize(cuboidIds.size());
+ readCuboidStatsFromSegments(cuboidIds, segmentList, statisticsMerged,
+ Maps.<Long, Double> newHashMapWithExpectedSize(cuboidIds.size()));
+ return statisticsMerged.isEmpty() ? null : statisticsMerged;
+ }
+
+ private static void readCuboidStatsFromSegments(Set<Long> cuboidSet, List<CubeSegment> segmentList,
+ final Map<Long, Long> statisticsMerged, final Map<Long, Double> sizeMerged) throws IOException {
+ if (segmentList == null || segmentList.isEmpty()) {
+ return;
+ }
+ int nSegment = segmentList.size();
+
+ Map<Long, HLLCounter> cuboidHLLMapMerged = Maps.newHashMapWithExpectedSize(cuboidSet.size());
+ Map<Long, Double> sizeMapMerged = Maps.newHashMapWithExpectedSize(cuboidSet.size());
+ for (CubeSegment pSegment : segmentList) {
+ CubeStatsReader pReader = new CubeStatsReader(pSegment, pSegment.getConfig());
+ Map<Long, HLLCounter> pHLLMap = pReader.getCuboidRowEstimatesHLLOrigin();
+ if (pHLLMap == null || pHLLMap.isEmpty()) {
+ logger.info("Cuboid Statistics for segment " + pSegment.getName() + " is not enabled.");
+ nSegment--;
+ continue;
+ }
+ Map<Long, Double> pSizeMap = pReader.getCuboidSizeMap();
+ for (Long pCuboid : cuboidSet) {
+ HLLCounter pInnerHLL = pHLLMap.get(pCuboid);
+ Preconditions.checkNotNull(pInnerHLL, "statistics should exist for cuboid " + pCuboid + " of segment "
+ + pSegment.getCubeDesc().getName() + "[" + pSegment.getName() + "]");
+ if (cuboidHLLMapMerged.get(pCuboid) != null) {
+ cuboidHLLMapMerged.get(pCuboid).merge(pInnerHLL);
+ } else {
+ cuboidHLLMapMerged.put(pCuboid, pInnerHLL);
+ }
+
+ Double pSize = sizeMapMerged.get(pCuboid);
+ sizeMapMerged.put(pCuboid, pSize == null ? pSizeMap.get(pCuboid) : pSizeMap.get(pCuboid) + pSize);
+ }
+ }
+
+ if (nSegment < 1) {
+ return;
+ }
+ for (Long pCuboid : cuboidSet) {
+ statisticsMerged.put(pCuboid, cuboidHLLMapMerged.get(pCuboid).getCountEstimate());
+ sizeMerged.put(pCuboid, sizeMapMerged.get(pCuboid));
+ }
+ }
+
+ public static Map<Long, Long> readCuboidStatsFromSegment(Set<Long> cuboidIds, CubeSegment cubeSegment)
+ throws IOException {
+ if (cubeSegment == null) {
+ logger.warn("The cube segment can not be " + null);
+ return null;
+ }
+
+ CubeStatsReader cubeStatsReader = new CubeStatsReader(cubeSegment, cubeSegment.getConfig());
+ if (cubeStatsReader.getCuboidRowEstimatesHLL() == null
+ || cubeStatsReader.getCuboidRowEstimatesHLL().isEmpty()) {
+ logger.info("Cuboid Statistics is not enabled.");
+ return null;
+ }
+
+ Map<Long, Long> cuboidsWithStatsAll = cubeStatsReader.getCuboidRowEstimatesHLL();
+ Map<Long, Long> cuboidsWithStats = Maps.newHashMapWithExpectedSize(cuboidIds.size());
+ for (Long cuboid : cuboidIds) {
+ Long rowEstimate = cuboidsWithStatsAll.get(cuboid);
+ if (rowEstimate == null) {
+ logger.warn("Cannot get the row count stats for cuboid " + cuboid);
+ } else {
+ cuboidsWithStats.put(cuboid, rowEstimate);
+ }
+ }
+ return cuboidsWithStats;
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f0bc691e/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/StatisticsDecisionUtil.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/StatisticsDecisionUtil.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/StatisticsDecisionUtil.java
new file mode 100644
index 0000000..4145226
--- /dev/null
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/StatisticsDecisionUtil.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.engine.mr.common;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.cube.CubeInstance;
+import org.apache.kylin.cube.CubeManager;
+import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.cube.CubeUpdate;
+import org.apache.kylin.engine.mr.CubingJob;
+import org.apache.kylin.metadata.model.MeasureDesc;
+import org.apache.kylin.metadata.model.SegmentStatusEnum;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class StatisticsDecisionUtil {
+ protected static final Logger logger = LoggerFactory.getLogger(StatisticsDecisionUtil.class);
+
+ public static void decideCubingAlgorithm(CubingJob cubingJob, CubeSegment seg) throws IOException {
+ CubeStatsReader cubeStats = new CubeStatsReader(seg, seg.getConfig());
+ decideCubingAlgorithm(cubingJob, seg, cubeStats.getMapperOverlapRatioOfFirstBuild(),
+ cubeStats.getMapperNumberOfFirstBuild());
+ }
+
+ public static void decideCubingAlgorithm(CubingJob cubingJob, CubeSegment seg, double mapperOverlapRatio,
+ int mapperNumber) throws IOException {
+ KylinConfig kylinConf = seg.getConfig();
+ String algPref = kylinConf.getCubeAlgorithm();
+ CubingJob.AlgorithmEnum alg;
+ if (mapperOverlapRatio == 0) { // no source records
+ alg = CubingJob.AlgorithmEnum.INMEM;
+ } else if (CubingJob.AlgorithmEnum.INMEM.name().equalsIgnoreCase(algPref)) {
+ alg = CubingJob.AlgorithmEnum.INMEM;
+ } else if (CubingJob.AlgorithmEnum.LAYER.name().equalsIgnoreCase(algPref)) {
+ alg = CubingJob.AlgorithmEnum.LAYER;
+ } else {
+ int memoryHungryMeasures = 0;
+ for (MeasureDesc measure : seg.getCubeDesc().getMeasures()) {
+ if (measure.getFunction().getMeasureType().isMemoryHungry()) {
+ logger.info("This cube has memory-hungry measure " + measure.getFunction().getExpression());
+ memoryHungryMeasures++;
+ }
+ }
+
+ if (memoryHungryMeasures > 0) {
+ alg = CubingJob.AlgorithmEnum.LAYER;
+ } else if ("random".equalsIgnoreCase(algPref)) { // for testing
+ alg = new Random().nextBoolean() ? CubingJob.AlgorithmEnum.INMEM : CubingJob.AlgorithmEnum.LAYER;
+ } else { // the default
+ int mapperNumLimit = kylinConf.getCubeAlgorithmAutoMapperLimit();
+ double overlapThreshold = kylinConf.getCubeAlgorithmAutoThreshold();
+ logger.info("mapperNumber for " + seg + " is " + mapperNumber + " and threshold is " + mapperNumLimit);
+ logger.info("mapperOverlapRatio for " + seg + " is " + mapperOverlapRatio + " and threshold is "
+ + overlapThreshold);
+
+ // in-mem cubing is good when
+ // 1) the cluster has enough mapper slots to run in parallel
+ // 2) the mapper overlap ratio is small, meaning the shuffle of in-mem MR has advantage
+ alg = (mapperNumber <= mapperNumLimit && mapperOverlapRatio <= overlapThreshold)//
+ ? CubingJob.AlgorithmEnum.INMEM
+ : CubingJob.AlgorithmEnum.LAYER;
+ }
+
+ }
+ logger.info("The cube algorithm for " + seg + " is " + alg);
+
+ cubingJob.setAlgorithm(alg);
+ }
+
+ public static void optimizeCubingPlan(CubeSegment segment) throws IOException {
+ CubeInstance cube = segment.getCubeInstance();
+ List<CubeSegment> readySegments = cube.getSegments(SegmentStatusEnum.READY);
+ if (readySegments.size() == 0 || (cube.getConfig().isCubePlannerEnabledForExistingCube()
+ && readySegments.size() == 1 && (readySegments.get(0).getSegRange().equals(segment.getSegRange())))) {
+ logger.info("It's able to trigger cuboid planner algorithm.");
+ } else {
+ return;
+ }
+
+ Map<Long, Long> recommendCuboidsWithStats = CuboidRecommenderUtil.getRecommendCuboidList(segment);
+ if (recommendCuboidsWithStats == null || recommendCuboidsWithStats.isEmpty()) {
+ return;
+ }
+
+ CubeUpdate cubeBuilder = new CubeUpdate(cube);
+ cubeBuilder.setCuboids(recommendCuboidsWithStats);
+ CubeManager.getInstance(cube.getConfig()).updateCube(cubeBuilder);
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f0bc691e/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java
index 28f99fb..7085cf0 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java
@@ -19,7 +19,6 @@
package org.apache.kylin.engine.mr.steps;
import java.io.IOException;
-import java.util.Random;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
@@ -30,14 +29,12 @@ import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.util.HadoopUtil;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.engine.mr.CubingJob;
-import org.apache.kylin.engine.mr.CubingJob.AlgorithmEnum;
import org.apache.kylin.engine.mr.common.BatchConstants;
-import org.apache.kylin.engine.mr.common.CubeStatsReader;
+import org.apache.kylin.engine.mr.common.StatisticsDecisionUtil;
import org.apache.kylin.job.exception.ExecuteException;
import org.apache.kylin.job.execution.AbstractExecutable;
import org.apache.kylin.job.execution.ExecutableContext;
import org.apache.kylin.job.execution.ExecuteResult;
-import org.apache.kylin.metadata.model.MeasureDesc;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -71,12 +68,15 @@ public class SaveStatisticsStep extends AbstractExecutable {
// put the statistics to metadata store
String statisticsFileName = newSegment.getStatisticsResourcePath();
rs.putResource(statisticsFileName, is, System.currentTimeMillis());
+
+ CubingJob cubingJob = (CubingJob) getManager()
+ .getJob(CubingExecutableUtil.getCubingJobId(this.getParams()));
+ StatisticsDecisionUtil.decideCubingAlgorithm(cubingJob, newSegment);
+ StatisticsDecisionUtil.optimizeCubingPlan(newSegment);
} finally {
IOUtils.closeStream(is);
}
- decideCubingAlgorithm(newSegment, kylinConf);
-
return new ExecuteResult(ExecuteResult.State.SUCCEED, "succeed");
} catch (IOException e) {
logger.error("fail to save cuboid statistics", e);
@@ -84,47 +84,4 @@ public class SaveStatisticsStep extends AbstractExecutable {
}
}
- private void decideCubingAlgorithm(CubeSegment seg, KylinConfig kylinConf) throws IOException {
- String algPref = kylinConf.getCubeAlgorithm();
- AlgorithmEnum alg;
- if (AlgorithmEnum.INMEM.name().equalsIgnoreCase(algPref)) {
- alg = AlgorithmEnum.INMEM;
- } else if (AlgorithmEnum.LAYER.name().equalsIgnoreCase(algPref)) {
- alg = AlgorithmEnum.LAYER;
- } else {
- int memoryHungryMeasures = 0;
- for (MeasureDesc measure : seg.getCubeDesc().getMeasures()) {
- if (measure.getFunction().getMeasureType().isMemoryHungry()) {
- logger.info("This cube has memory-hungry measure " + measure.getFunction().getExpression());
- memoryHungryMeasures++;
- }
- }
-
- if (memoryHungryMeasures > 0) {
- alg = AlgorithmEnum.LAYER;
- } else if ("random".equalsIgnoreCase(algPref)) { // for testing
- alg = new Random().nextBoolean() ? AlgorithmEnum.INMEM : AlgorithmEnum.LAYER;
- } else { // the default
- CubeStatsReader cubeStats = new CubeStatsReader(seg, kylinConf);
- int mapperNumber = cubeStats.getMapperNumberOfFirstBuild();
- int mapperNumLimit = kylinConf.getCubeAlgorithmAutoMapperLimit();
- double mapperOverlapRatio = cubeStats.getMapperOverlapRatioOfFirstBuild();
- double overlapThreshold = kylinConf.getCubeAlgorithmAutoThreshold();
- logger.info("mapperNumber for " + seg + " is " + mapperNumber + " and threshold is " + mapperNumLimit);
- logger.info("mapperOverlapRatio for " + seg + " is " + mapperOverlapRatio + " and threshold is " + overlapThreshold);
-
- // in-mem cubing is good when
- // 1) the cluster has enough mapper slots to run in parallel
- // 2) the mapper overlap ratio is small, meaning the shuffle of in-mem MR has advantage
- alg = (mapperNumber <= mapperNumLimit && mapperOverlapRatio <= overlapThreshold)//
- ? AlgorithmEnum.INMEM : AlgorithmEnum.LAYER;
- }
-
- }
- logger.info("The cube algorithm for " + seg + " is " + alg);
-
- CubingJob cubingJob = (CubingJob) getManager().getJob(CubingExecutableUtil.getCubingJobId(this.getParams()));
- cubingJob.setAlgorithm(alg);
- }
-
}
[11/13] kylin git commit: KYLIN-2730 Introduce genetic algorithm for
cube planner
Posted by li...@apache.org.
KYLIN-2730 Introduce genetic algorithm for cube planner
Signed-off-by: Li Yang <li...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7b6606ad
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7b6606ad
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7b6606ad
Branch: refs/heads/pr70
Commit: 7b6606ad7ff6e89595c3db791a80607fe816c423
Parents: 01fe425
Author: Zhong <nj...@apache.org>
Authored: Tue Aug 22 14:59:02 2017 +0800
Committer: Li Yang <li...@apache.org>
Committed: Sat Sep 23 18:16:48 2017 +0800
----------------------------------------------------------------------
.../algorithm/generic/BitsChromosome.java | 133 +++++++++
.../generic/CombinedStoppingCondition.java | 43 +++
.../cuboid/algorithm/generic/CuboidEncoder.java | 44 +++
.../algorithm/generic/GeneticAlgorithm.java | 283 +++++++++++++++++++
.../generic/RouletteWheelSelection.java | 61 ++++
.../algorithm/generic/lib/BitsMutation.java | 59 ++++
.../algorithm/generic/lib/Chromosome.java | 104 +++++++
.../lib/ChromosomeMismatchException.java | 48 ++++
.../algorithm/generic/lib/ChromosomePair.java | 67 +++++
.../algorithm/generic/lib/CrossoverPolicy.java | 37 +++
.../generic/lib/ElitisticListPopulation.java | 117 ++++++++
.../cuboid/algorithm/generic/lib/Fitness.java | 33 +++
.../generic/lib/FixedGenerationCount.java | 71 +++++
.../algorithm/generic/lib/ListPopulation.java | 223 +++++++++++++++
.../algorithm/generic/lib/MutationPolicy.java | 34 +++
.../generic/lib/OnePointCrossover.java | 126 +++++++++
.../algorithm/generic/lib/Population.java | 59 ++++
.../algorithm/generic/lib/SelectionPolicy.java | 33 +++
.../generic/lib/StoppingCondition.java | 34 +++
.../generic/lib/TournamentSelection.java | 114 ++++++++
.../cuboid/algorithm/GeneticAlgorithmTest.java | 57 ++++
21 files changed, 1780 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/7b6606ad/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/BitsChromosome.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/BitsChromosome.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/BitsChromosome.java
new file mode 100755
index 0000000..6445ca6
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/BitsChromosome.java
@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.cube.cuboid.algorithm.generic;
+
+import java.util.BitSet;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.kylin.cube.cuboid.algorithm.BenefitPolicy;
+import org.apache.kylin.cube.cuboid.algorithm.CuboidBenefitModel;
+import org.apache.kylin.cube.cuboid.algorithm.generic.lib.Chromosome;
+import org.apache.kylin.cube.cuboid.algorithm.CuboidStats;
+
+import com.google.common.collect.Sets;
+
+public class BitsChromosome extends Chromosome {
+
+ private BitSet key;
+ private int length;
+ private BenefitPolicy benefitPolicy;
+ private CuboidStats cuboidStats;
+ private CuboidEncoder cuboidEncoder;
+ private double spaceLimit;
+ private double spaceCost;
+ private double fitness = -1;
+
+ public BitsChromosome(final BitSet key, final BenefitPolicy benefitPolicy, final CuboidStats cuboidStats,
+ final double spaceLimit) {
+ super();
+ this.key = key;
+ this.length = cuboidStats.getAllCuboidsForSelection().size();
+ this.benefitPolicy = benefitPolicy.getInstance();
+ this.cuboidStats = cuboidStats;
+ this.cuboidEncoder = new CuboidEncoder(cuboidStats);
+ this.spaceLimit = spaceLimit;
+ initSpaceCost();
+ }
+
+ public BitsChromosome newBitsChromosome(BitSet newkey) {
+ return new BitsChromosome(newkey, this.benefitPolicy, this.cuboidStats, this.spaceLimit);
+ }
+
+ private void initSpaceCost() {
+ spaceCost = 0;
+ List<Long> remainingCuboids = cuboidEncoder.toCuboidList(key);
+ for (Long cuboid : remainingCuboids) {
+ spaceCost += cuboidStats.getCuboidSize(cuboid);
+ }
+ }
+
+ public BitSet getKey() {
+ return key;
+ }
+
+ public int getLength() {
+ return length;
+ }
+
+ public CuboidEncoder getCuboidEncoder() {
+ return cuboidEncoder;
+ }
+
+ @Override
+ public double fitness() {
+ if (fitness == -1) {
+ fitness = calculateFitness();
+ }
+ return fitness;
+ }
+
+ @Override
+ protected boolean isSame(final Chromosome another) {
+ return this.equals(another);
+ }
+
+ private synchronized double calculateFitness() {
+ List<Long> remainingCuboids = cuboidEncoder.toCuboidList(key);
+ Set<Long> selectedCuboidSets = Sets.newHashSet();
+ selectedCuboidSets.addAll(cuboidStats.getAllCuboidsForMandatory());
+
+ CuboidBenefitModel.BenefitModel benefitModel = benefitPolicy.calculateBenefitTotal(remainingCuboids, selectedCuboidSets);
+ double totalBenefit = benefitModel.getBenefit();
+ if (spaceCost > spaceLimit) {
+ totalBenefit = totalBenefit * spaceLimit / spaceCost;
+ }
+ return totalBenefit;
+ }
+
+ @Override
+ public int hashCode() {
+ final int prime = 31;
+ int result = 1;
+ result = prime * result + ((key == null) ? 0 : key.hashCode());
+ result = prime * result + length;
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj)
+ return true;
+ if (obj == null)
+ return false;
+ if (getClass() != obj.getClass())
+ return false;
+ BitsChromosome other = (BitsChromosome) obj;
+ if (length != other.length) {
+ return false;
+ }
+ if (key == null) {
+ if (other.key != null)
+ return false;
+ } else if (!key.equals(other.key))
+ return false;
+ return true;
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/7b6606ad/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/CombinedStoppingCondition.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/CombinedStoppingCondition.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/CombinedStoppingCondition.java
new file mode 100755
index 0000000..bc4bb99
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/CombinedStoppingCondition.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.cube.cuboid.algorithm.generic;
+
+import org.apache.kylin.cube.cuboid.algorithm.generic.lib.Population;
+import org.apache.kylin.cube.cuboid.algorithm.generic.lib.StoppingCondition;
+
+import java.util.List;
+
+public class CombinedStoppingCondition implements StoppingCondition {
+ private List<StoppingCondition> conditions;
+
+ public CombinedStoppingCondition(List<StoppingCondition> conditions) {
+ this.conditions = conditions;
+ }
+
+ @Override
+ public boolean isSatisfied(Population population) {
+ for (StoppingCondition condition : conditions) {
+ if (condition.isSatisfied(population)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/7b6606ad/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/CuboidEncoder.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/CuboidEncoder.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/CuboidEncoder.java
new file mode 100755
index 0000000..7e48413
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/CuboidEncoder.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.cube.cuboid.algorithm.generic;
+
+import java.util.BitSet;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.kylin.cube.cuboid.algorithm.CuboidStats;
+
+import com.google.common.collect.Lists;
+
+public class CuboidEncoder {
+ private List<Long> selectionCuboids;
+
+ public CuboidEncoder(CuboidStats cuboidStats) {
+ selectionCuboids = Lists.newArrayList(cuboidStats.getAllCuboidsForSelection());
+ Collections.sort(selectionCuboids, Collections.reverseOrder());
+ }
+
+ public List<Long> toCuboidList(BitSet bits) {
+ List<Long> cuboids = Lists.newArrayListWithCapacity(bits.cardinality());
+ for (int i = bits.nextSetBit(0); i >= 0; i = bits.nextSetBit(i + 1)) {
+ cuboids.add(selectionCuboids.get(i));
+ }
+ return cuboids;
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/7b6606ad/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/GeneticAlgorithm.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/GeneticAlgorithm.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/GeneticAlgorithm.java
new file mode 100755
index 0000000..e2545b9
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/GeneticAlgorithm.java
@@ -0,0 +1,283 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.cube.cuboid.algorithm.generic;
+
+import java.util.BitSet;
+import java.util.List;
+import java.util.Random;
+
+import org.apache.kylin.cube.cuboid.algorithm.AbstractRecommendAlgorithm;
+import org.apache.kylin.cube.cuboid.algorithm.BenefitPolicy;
+import org.apache.kylin.cube.cuboid.algorithm.generic.lib.BitsMutation;
+import org.apache.kylin.cube.cuboid.algorithm.generic.lib.Chromosome;
+import org.apache.kylin.cube.cuboid.algorithm.generic.lib.ChromosomePair;
+import org.apache.kylin.cube.cuboid.algorithm.generic.lib.CrossoverPolicy;
+import org.apache.kylin.cube.cuboid.algorithm.generic.lib.ElitisticListPopulation;
+import org.apache.kylin.cube.cuboid.algorithm.generic.lib.FixedGenerationCount;
+import org.apache.kylin.cube.cuboid.algorithm.generic.lib.MutationPolicy;
+import org.apache.kylin.cube.cuboid.algorithm.generic.lib.OnePointCrossover;
+import org.apache.kylin.cube.cuboid.algorithm.generic.lib.Population;
+import org.apache.kylin.cube.cuboid.algorithm.generic.lib.SelectionPolicy;
+import org.apache.kylin.cube.cuboid.algorithm.generic.lib.StoppingCondition;
+import org.apache.kylin.cube.cuboid.algorithm.CuboidStats;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.collect.Lists;
+
+/**
+ * Implementation of a genetic algorithm to recommend a list of cuboids. All factors that govern the processing
+ * of the algorithm can be configured.
+ *
+ */
+public class GeneticAlgorithm extends AbstractRecommendAlgorithm {
+
+ private static final Logger logger = LoggerFactory.getLogger(GeneticAlgorithm.class);
+ public static ThreadLocal<Random> RANDGEN = new ThreadLocal<Random>() {
+ @Override
+ protected Random initialValue() {
+ return new Random(System.currentTimeMillis() * Thread.currentThread().getId());
+ }
+ };
+ /** the rate of crossover for the algorithm. */
+ private final double crossoverRate = 0.9;
+ /** the rate of mutation for the algorithm. */
+ private final double mutationRate = 0.001;
+ /** the init population size. */
+ private final int populationSize = 500;
+ /** the max population size. */
+ private final int maxPopulationSize = 510;
+ private CrossoverPolicy crossoverPolicy;
+ private MutationPolicy mutationPolicy;
+ private SelectionPolicy selectionPolicy;
+ private CuboidEncoder cuboidEncoder;
+ /** the number of generations evolved to reach {@link StoppingCondition} in the last run. */
+ private int generationsEvolved = 0;
+
+ public GeneticAlgorithm(final long timeout, BenefitPolicy benefitPolicy, CuboidStats cuboidStats) {
+ super(timeout, benefitPolicy, cuboidStats);
+ this.crossoverPolicy = new OnePointCrossover();
+ this.mutationPolicy = new BitsMutation();
+ this.selectionPolicy = new RouletteWheelSelection();
+
+ this.cuboidEncoder = new CuboidEncoder(getCuboidStats());
+ }
+
+ @Override
+ public List<Long> recommend(double expansionRate) {
+ double spaceLimit = getCuboidStats().getBaseCuboidSize() * expansionRate;
+ return start(spaceLimit);
+ }
+
+ @Override
+ public List<Long> start(double maxSpaceLimit) {
+ logger.info("Genetic Algorithm started.");
+
+ getBenefitPolicy().initBeforeStart();
+
+ //Initial mandatory cuboids
+ double remainingSpace = maxSpaceLimit;
+ for (Long mandatoryOne : getCuboidStats().getAllCuboidsForMandatory()) {
+ if (getCuboidStats().getCuboidSize(mandatoryOne) != null) {
+ remainingSpace -= getCuboidStats().getCuboidSize(mandatoryOne);
+ }
+ }
+
+ //Generate a population randomly
+ Population initial = initRandomPopulation(remainingSpace);
+
+ //Set stopping condition
+ List<StoppingCondition> conditions = Lists.newArrayList();
+ conditions.add(new FixedGenerationCount(550));
+ CombinedStoppingCondition stopCondition = new CombinedStoppingCondition(conditions);
+
+ //Start the evolution
+ Population current = evolve(initial, stopCondition);
+ BitsChromosome chromosome = (BitsChromosome) current.getFittestChromosome();
+ logger.info("Genetic Algorithm finished.");
+ List<Long> finalList = Lists.newArrayList();
+ finalList.addAll(getCuboidStats().getAllCuboidsForMandatory());
+ finalList.addAll(cuboidEncoder.toCuboidList(chromosome.getKey()));
+
+ double totalSpace = 0;
+ if (logger.isInfoEnabled()) {
+ for (Long cuboid : finalList) {
+ Double unitSpace = getCuboidStats().getCuboidSize(cuboid);
+ if (unitSpace != null) {
+ logger.info(String.format("cuboidId %d and Space: %f", cuboid, unitSpace));
+ totalSpace += unitSpace;
+ } else {
+ logger.info(String.format("mandatory cuboidId %d", cuboid));
+ }
+ }
+ logger.info("Total Space:" + totalSpace);
+ logger.info("Space Expansion Rate:" + totalSpace / getCuboidStats().getBaseCuboidSize());
+ }
+ return finalList;
+ }
+
+ protected Population initRandomPopulation(double maxSpaceLimit) {
+ List<Chromosome> chromosomeList = Lists.newArrayListWithCapacity(populationSize);
+ List<Long> cuboidsForSelection = Lists.newArrayList(getCuboidStats().getAllCuboidsForSelection());
+ int selectionSize = cuboidsForSelection.size();
+
+ while (chromosomeList.size() < populationSize) {
+ BitSet bitSetForSelection = new BitSet(selectionSize);
+
+ //Initialize selection genes
+ double totalSpace = 0;
+ while (totalSpace < maxSpaceLimit) {
+ int j = RANDGEN.get().nextInt(selectionSize);
+ if (!bitSetForSelection.get(j)) {
+ totalSpace += getCuboidStats().getCuboidSize(cuboidsForSelection.get(j));
+ bitSetForSelection.set(j);
+ }
+ }
+
+ Chromosome chromosome = new BitsChromosome(bitSetForSelection, getBenefitPolicy(), getCuboidStats(),
+ maxSpaceLimit);
+ chromosomeList.add(chromosome);
+ }
+ return new ElitisticListPopulation(chromosomeList, maxPopulationSize, 0.8);
+ }
+
+ /**
+ * Evolve the given population. Evolution stops when the stopping condition
+ * is satisfied. Updates the {@link #getGenerationsEvolved() generationsEvolved}
+ * property with the number of generations evolved before the StoppingCondition
+ * is satisfied.
+ *
+ * @param initial the initial, seed population.
+ * @param condition the stopping condition used to stop evolution.
+ * @return the population that satisfies the stopping condition.
+ */
+ protected Population evolve(final Population initial, final StoppingCondition condition) {
+ Population current = initial;
+ generationsEvolved = 0;
+ while (!condition.isSatisfied(current) && (!shouldCancel())) {
+ current = nextGeneration(current);
+ generationsEvolved++;
+ logger.info("Generations evolved count:" + generationsEvolved);
+ }
+ return current;
+ }
+
+ /**
+ * Evolve the given population into the next generation.
+ * <p>
+ * <ol>
+ * <li>Get nextGeneration population to fill from <code>current</code>
+ * generation, using its nextGeneration method</li>
+ * <li>Loop until new generation is filled:</li>
+ * <ul><li>Apply configured SelectionPolicy to select a pair of parents
+ * from <code>current</code></li>
+ * <li>With probability = {@link #getCrossoverRate()}, apply
+ * configured {@link CrossoverPolicy} to parents</li>
+ * <li>With probability = {@link #getMutationRate()}, apply
+ * configured {@link MutationPolicy} to each of the offspring</li>
+ * <li>Add offspring individually to nextGeneration,
+ * space permitting</li>
+ * </ul>
+ * <li>Return nextGeneration</li>
+ * </ol>
+ *
+ * @param current the current population.
+ * @return the population for the next generation.
+ */
+ protected Population nextGeneration(final Population current) {
+ Population nextGeneration = current.nextGeneration();
+
+ while (nextGeneration.getPopulationSize() < nextGeneration.getPopulationLimit()) {
+ // select parent chromosomes
+ ChromosomePair pair = getSelectionPolicy().select(current);
+
+ // crossover?
+ if (RANDGEN.get().nextDouble() < getCrossoverRate()) {
+ // apply crossover policy to create two offspring
+ pair = getCrossoverPolicy().crossover(pair.getFirst(), pair.getSecond());
+ }
+
+ // mutation?
+ if (RANDGEN.get().nextDouble() < getMutationRate()) {
+ // apply mutation policy to the chromosomes
+ pair = new ChromosomePair(getMutationPolicy().mutate(pair.getFirst()),
+ getMutationPolicy().mutate(pair.getSecond()));
+ }
+
+ // add the first chromosome to the population
+ nextGeneration.addChromosome(pair.getFirst());
+ // is there still a place for the second chromosome?
+ if (nextGeneration.getPopulationSize() < nextGeneration.getPopulationLimit()) {
+ // add the second chromosome to the population
+ nextGeneration.addChromosome(pair.getSecond());
+ }
+ }
+ return nextGeneration;
+ }
+
+ /**
+ * Returns the crossover policy.
+ * @return crossover policy
+ */
+ public CrossoverPolicy getCrossoverPolicy() {
+ return crossoverPolicy;
+ }
+
+ /**
+ * Returns the crossover rate.
+ * @return crossover rate
+ */
+ public double getCrossoverRate() {
+ return crossoverRate;
+ }
+
+ /**
+ * Returns the mutation policy.
+ * @return mutation policy
+ */
+ public MutationPolicy getMutationPolicy() {
+ return mutationPolicy;
+ }
+
+ /**
+ * Returns the mutation rate.
+ * @return mutation rate
+ */
+ public double getMutationRate() {
+ return mutationRate;
+ }
+
+ /**
+ * Returns the selection policy.
+ * @return selection policy
+ */
+ public SelectionPolicy getSelectionPolicy() {
+ return selectionPolicy;
+ }
+
+ /**
+ * Returns the number of generations evolved to reach {@link StoppingCondition} in the last run.
+ *
+ * @return number of generations evolved
+ */
+ public int getGenerationsEvolved() {
+ return generationsEvolved;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/7b6606ad/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/RouletteWheelSelection.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/RouletteWheelSelection.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/RouletteWheelSelection.java
new file mode 100755
index 0000000..db84171
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/RouletteWheelSelection.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.cube.cuboid.algorithm.generic;
+
+import java.util.List;
+
+import org.apache.kylin.cube.cuboid.algorithm.generic.lib.Chromosome;
+import org.apache.kylin.cube.cuboid.algorithm.generic.lib.ChromosomePair;
+import org.apache.kylin.cube.cuboid.algorithm.generic.lib.ListPopulation;
+import org.apache.kylin.cube.cuboid.algorithm.generic.lib.Population;
+import org.apache.kylin.cube.cuboid.algorithm.generic.lib.SelectionPolicy;
+
+import com.google.common.collect.Lists;
+
+public class RouletteWheelSelection implements SelectionPolicy {
+
+ @Override
+ public ChromosomePair select(Population population) throws IllegalArgumentException {
+ // create a copy of the chromosome list
+ List<Chromosome> chromosomes = Lists.newArrayList(((ListPopulation) population).getChromosomes());
+
+ double maxFitness = 0;
+ double totalFitness = 0;
+ for (Chromosome o : chromosomes) {
+ double fitness = o.getFitness();
+ totalFitness += fitness;
+ if (fitness > maxFitness) {
+ maxFitness = fitness;
+ }
+ }
+ return new ChromosomePair(rouletteWheel(chromosomes, totalFitness), rouletteWheel(chromosomes, totalFitness));
+ }
+
+ private Chromosome rouletteWheel(final List<Chromosome> chromosomes, final double totalFitness) {
+ float rnd = (float) (GeneticAlgorithm.RANDGEN.get().nextDouble() * totalFitness);
+ float runningScore = 0;
+ for (Chromosome o : chromosomes) {
+ if (rnd >= runningScore && rnd <= runningScore + o.getFitness()) {
+ return o;
+ }
+ runningScore += o.getFitness();
+ }
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/7b6606ad/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/lib/BitsMutation.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/lib/BitsMutation.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/lib/BitsMutation.java
new file mode 100755
index 0000000..b349eb6
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/lib/BitsMutation.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.cube.cuboid.algorithm.generic.lib;
+
+import java.util.BitSet;
+
+import org.apache.kylin.cube.cuboid.algorithm.generic.BitsChromosome;
+import org.apache.kylin.cube.cuboid.algorithm.generic.GeneticAlgorithm;
+
+/**
+ * Mutation for {@link BitsChromosome}s. Randomly changes one gene.
+ *
+ */
+public class BitsMutation implements MutationPolicy {
+
+ /**
+ * Mutate the given chromosome. Randomly changes one gene.
+ *
+ * @param original the original chromosome.
+ * @return the mutated chromosome.
+ * @throws IllegalArgumentException if <code>original</code> is not an instance of {@link BitsChromosome}.
+ */
+ public Chromosome mutate(Chromosome original) throws IllegalArgumentException {
+ if (!(original instanceof BitsChromosome)) {
+ throw new IllegalArgumentException("Chromosome " + original.getClass() + " must be of type BitsChromosome.");
+ }
+
+ BitsChromosome origChrom = (BitsChromosome) original;
+ BitSet newNey = (BitSet) origChrom.getKey().clone();
+
+ // randomly select a gene
+ int geneIndex = getMutationGeneIndex(origChrom);
+ // change it
+ boolean value = newNey.get(geneIndex);
+ newNey.set(geneIndex, !value);
+ Chromosome newChrom = origChrom.newBitsChromosome(newNey);
+ return newChrom;
+ }
+
+ protected int getMutationGeneIndex(BitsChromosome origChrom) {
+ return GeneticAlgorithm.RANDGEN.get().nextInt(origChrom.getLength());
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/7b6606ad/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/lib/Chromosome.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/lib/Chromosome.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/lib/Chromosome.java
new file mode 100755
index 0000000..f8fa489
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/lib/Chromosome.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.cube.cuboid.algorithm.generic.lib;
+
+/**
+ * Individual in a population. Chromosomes are compared based on their fitness.
+ * <p>
+ * The chromosomes are IMMUTABLE, and so their fitness is also immutable and
+ * therefore it can be cached.
+ *
+ */
+public abstract class Chromosome implements Comparable<Chromosome>, Fitness {
+ /** Value assigned when no fitness has been computed yet. */
+ private static final double NO_FITNESS = Double.NEGATIVE_INFINITY;
+
+ /** Cached value of the fitness of this chromosome. */
+ private double fitness = NO_FITNESS;
+
+ /**
+ * Access the fitness of this chromosome. The bigger the fitness, the better the chromosome.
+ * <p>
+ * Computation of fitness is usually very time-consuming task, therefore the fitness is cached.
+ *
+ * @return the fitness
+ */
+ public double getFitness() {
+ if (this.fitness == NO_FITNESS) {
+ // no cache - compute the fitness
+ this.fitness = fitness();
+ }
+ return this.fitness;
+ }
+
+ /**
+ * Compares two chromosomes based on their fitness. The bigger the fitness, the better the chromosome.
+ *
+ * @param another another chromosome to compare
+ * @return
+ * <ul>
+ * <li>-1 if <code>another</code> is better than <code>this</code></li>
+ * <li>1 if <code>another</code> is worse than <code>this</code></li>
+ * <li>0 if the two chromosomes have the same fitness</li>
+ * </ul>
+ */
+ public int compareTo(final Chromosome another) {
+ return Double.compare(getFitness(), another.getFitness());
+ }
+
+ /**
+ * Returns <code>true</code> iff <code>another</code> has the same representation and therefore the same fitness. By
+ * default, it returns false -- override it in your implementation if you need it.
+ *
+ * @param another chromosome to compare
+ * @return true if <code>another</code> is equivalent to this chromosome
+ */
+ protected boolean isSame(final Chromosome another) {
+ return false;
+ }
+
+ /**
+ * Searches the <code>population</code> for another chromosome with the same representation. If such chromosome is
+ * found, it is returned, if no such chromosome exists, returns <code>null</code>.
+ *
+ * @param population Population to search
+ * @return Chromosome with the same representation, or <code>null</code> if no such chromosome exists.
+ */
+ protected Chromosome findSameChromosome(final Population population) {
+ for (Chromosome anotherChr : population) {
+ if (this.isSame(anotherChr)) {
+ return anotherChr;
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Searches the population for a chromosome representing the same solution, and if it finds one,
+ * updates the fitness to its value.
+ *
+ * @param population Population to search
+ */
+ public void searchForFitnessUpdate(final Population population) {
+ Chromosome sameChromosome = findSameChromosome(population);
+ if (sameChromosome != null) {
+ fitness = sameChromosome.getFitness();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/7b6606ad/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/lib/ChromosomeMismatchException.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/lib/ChromosomeMismatchException.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/lib/ChromosomeMismatchException.java
new file mode 100755
index 0000000..51d5cb3
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/lib/ChromosomeMismatchException.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.cube.cuboid.algorithm.generic.lib;
+
+/**
+ * Exception to be thrown when two Chromosomes length differ.
+ *
+ */
+public class ChromosomeMismatchException extends IllegalArgumentException {
+ private static final long serialVersionUID = -7483865132286153255L;
+
+ private final int length;
+
+ /**
+ * Construct an exception from the mismatched chromosomes.
+ *
+ * @param errorMsg error info.
+ * @param wrong Wrong length.
+ * @param expected Expected length.
+ */
+ public ChromosomeMismatchException(String errorMsg, int wrong, int expected) {
+ super(errorMsg);
+ length = expected;
+ }
+
+ /**
+ * @return the expected length.
+ */
+ public int getLength() {
+ return length;
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/7b6606ad/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/lib/ChromosomePair.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/lib/ChromosomePair.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/lib/ChromosomePair.java
new file mode 100755
index 0000000..05da4b5
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/lib/ChromosomePair.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.cube.cuboid.algorithm.generic.lib;
+
+/**
+ * A pair of {@link Chromosome} objects.
+ *
+ */
+public class ChromosomePair {
+ /** the first chromosome in the pair. */
+ private final Chromosome first;
+
+ /** the second chromosome in the pair. */
+ private final Chromosome second;
+
+ /**
+ * Create a chromosome pair.
+ *
+ * @param c1 the first chromosome.
+ * @param c2 the second chromosome.
+ */
+ public ChromosomePair(final Chromosome c1, final Chromosome c2) {
+ super();
+ first = c1;
+ second = c2;
+ }
+
+ /**
+ * Access the first chromosome.
+ *
+ * @return the first chromosome.
+ */
+ public Chromosome getFirst() {
+ return first;
+ }
+
+ /**
+ * Access the second chromosome.
+ *
+ * @return the second chromosome.
+ */
+ public Chromosome getSecond() {
+ return second;
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public String toString() {
+ return String.format("(%s,%s)", getFirst(), getSecond());
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/7b6606ad/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/lib/CrossoverPolicy.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/lib/CrossoverPolicy.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/lib/CrossoverPolicy.java
new file mode 100755
index 0000000..fa287cf
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/lib/CrossoverPolicy.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.cube.cuboid.algorithm.generic.lib;
+
+/**
+ * Policy used to create a pair of new chromosomes by performing a crossover
+ * operation on a source pair of chromosomes.
+ *
+ */
+public interface CrossoverPolicy {
+
+ /**
+ * Perform a crossover operation on the given chromosomes.
+ *
+ * @param first the first chromosome.
+ * @param second the second chromosome.
+ * @return the pair of new chromosomes that resulted from the crossover.
+ * @throws IllegalArgumentException if the given chromosomes are not compatible with this {@link CrossoverPolicy}
+ */
+ ChromosomePair crossover(Chromosome first, Chromosome second) throws IllegalArgumentException;
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/7b6606ad/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/lib/ElitisticListPopulation.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/lib/ElitisticListPopulation.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/lib/ElitisticListPopulation.java
new file mode 100755
index 0000000..f66e941
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/lib/ElitisticListPopulation.java
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.cube.cuboid.algorithm.generic.lib;
+
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.commons.math3.exception.NotPositiveException;
+import org.apache.commons.math3.exception.NullArgumentException;
+import org.apache.commons.math3.exception.NumberIsTooLargeException;
+import org.apache.commons.math3.exception.OutOfRangeException;
+import org.apache.commons.math3.exception.util.LocalizedFormats;
+import org.apache.commons.math3.util.FastMath;
+
+/**
+ * Population of chromosomes which uses elitism (certain percentage of the best
+ * chromosomes is directly copied to the next generation).
+ *
+ */
+public class ElitisticListPopulation extends ListPopulation {
+
+ /** percentage of chromosomes copied to the next generation */
+ private double elitismRate = 0.9;
+
+ /**
+ * Creates a new {@link ElitisticListPopulation} instance.
+ *
+ * @param chromosomes list of chromosomes in the population
+ * @param populationLimit maximal size of the population
+ * @param elitismRate how many best chromosomes will be directly transferred to the next generation [in %]
+ * @throws NullArgumentException if the list of chromosomes is {@code null}
+ * @throws NotPositiveException if the population limit is not a positive number (< 1)
+ * @throws NumberIsTooLargeException if the list of chromosomes exceeds the population limit
+ * @throws OutOfRangeException if the elitism rate is outside the [0, 1] range
+ */
+ public ElitisticListPopulation(final List<Chromosome> chromosomes, final int populationLimit,
+ final double elitismRate)
+ throws NullArgumentException, NotPositiveException, NumberIsTooLargeException, OutOfRangeException {
+
+ super(chromosomes, populationLimit);
+ setElitismRate(elitismRate);
+ }
+
+ /**
+ * Creates a new {@link ElitisticListPopulation} instance and initializes its inner chromosome list.
+ *
+ * @param populationLimit maximal size of the population
+ * @param elitismRate how many best chromosomes will be directly transferred to the next generation [in %]
+ * @throws NotPositiveException if the population limit is not a positive number (< 1)
+ * @throws OutOfRangeException if the elitism rate is outside the [0, 1] range
+ */
+ public ElitisticListPopulation(final int populationLimit, final double elitismRate)
+ throws NotPositiveException, OutOfRangeException {
+
+ super(populationLimit);
+ setElitismRate(elitismRate);
+ }
+
+ /**
+ * Start the population for the next generation. The <code>{@link #elitismRate}</code>
+ * percents of the best chromosomes are directly copied to the next generation.
+ *
+ * @return the beginnings of the next generation.
+ */
+ public Population nextGeneration() {
+ // initialize a new generation with the same parameters
+ ElitisticListPopulation nextGeneration = new ElitisticListPopulation(getPopulationLimit(), getElitismRate());
+
+ final List<Chromosome> oldChromosomes = getChromosomeList();
+ Collections.sort(oldChromosomes);
+
+ // index of the last "not good enough" chromosome
+ int boundIndex = (int) FastMath.ceil((1.0 - getElitismRate()) * oldChromosomes.size());
+ for (int i = boundIndex; i < oldChromosomes.size(); i++) {
+ nextGeneration.addChromosome(oldChromosomes.get(i));
+ }
+ return nextGeneration;
+ }
+
+ /**
+ * Access the elitism rate.
+ * @return the elitism rate
+ */
+ public double getElitismRate() {
+ return this.elitismRate;
+ }
+
+ /**
+ * Sets the elitism rate, i.e. how many best chromosomes will be directly transferred to the next generation [in %].
+ *
+ * @param elitismRate how many best chromosomes will be directly transferred to the next generation [in %]
+ * @throws OutOfRangeException if the elitism rate is outside the [0, 1] range
+ */
+ public void setElitismRate(final double elitismRate) throws OutOfRangeException {
+ if (elitismRate < 0 || elitismRate > 1) {
+ throw new OutOfRangeException(LocalizedFormats.ELITISM_RATE, elitismRate, 0, 1);
+ }
+ this.elitismRate = elitismRate;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/7b6606ad/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/lib/Fitness.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/lib/Fitness.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/lib/Fitness.java
new file mode 100755
index 0000000..83f4b65
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/lib/Fitness.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.cube.cuboid.algorithm.generic.lib;
+
+/**
+ * Fitness of a chromosome.
+ *
+ */
+public interface Fitness {
+
+ /**
+ * Compute the fitness.
+ * @return fitness
+ */
+ double fitness();
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/7b6606ad/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/lib/FixedGenerationCount.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/lib/FixedGenerationCount.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/lib/FixedGenerationCount.java
new file mode 100755
index 0000000..c0c8316
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/lib/FixedGenerationCount.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.cube.cuboid.algorithm.generic.lib;
+
+import org.apache.commons.math3.exception.NumberIsTooSmallException;
+
+/**
+ * Stops after a fixed number of generations. Each time {@link #isSatisfied(Population)} is invoked, a generation
+ * counter is incremented. Once the counter reaches the configured <code>maxGenerations</code> value,
+ * {@link #isSatisfied(Population)} returns true.
+ *
+ */
+public class FixedGenerationCount implements StoppingCondition {
+ /** Maximum number of generations (stopping criteria) */
+ private final int maxGenerations;
+ /** Number of generations that have passed */
+ private int numGenerations = 0;
+
+ /**
+ * Create a new FixedGenerationCount instance.
+ *
+ * @param maxGenerations number of generations to evolve
+ * @throws NumberIsTooSmallException if the number of generations is < 1
+ */
+ public FixedGenerationCount(final int maxGenerations) throws NumberIsTooSmallException {
+ if (maxGenerations <= 0) {
+ throw new NumberIsTooSmallException(maxGenerations, 1, true);
+ }
+ this.maxGenerations = maxGenerations;
+ }
+
+ /**
+ * Determine whether or not the given number of generations have passed. Increments the number of generations
+ * counter if the maximum has not been reached.
+ *
+ * @param population ignored (no impact on result)
+ * @return <code>true</code> IFF the maximum number of generations has been exceeded
+ */
+ public boolean isSatisfied(final Population population) {
+ if (this.numGenerations < this.maxGenerations) {
+ numGenerations++;
+ return false;
+ }
+ return true;
+ }
+
+ /**
+ * Returns the number of generations that have already passed.
+ * @return the number of generations that have passed
+ */
+ public int getNumGenerations() {
+ return numGenerations;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/7b6606ad/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/lib/ListPopulation.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/lib/ListPopulation.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/lib/ListPopulation.java
new file mode 100755
index 0000000..c146274
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/lib/ListPopulation.java
@@ -0,0 +1,223 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.cube.cuboid.algorithm.generic.lib;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.commons.math3.exception.NotPositiveException;
+import org.apache.commons.math3.exception.NullArgumentException;
+import org.apache.commons.math3.exception.NumberIsTooLargeException;
+import org.apache.commons.math3.exception.NumberIsTooSmallException;
+import org.apache.commons.math3.exception.util.LocalizedFormats;
+
+/**
+ * Population of chromosomes represented by a {@link List}.
+ *
+ */
+public abstract class ListPopulation implements Population {
+
+ /** List of chromosomes */
+ private List<Chromosome> chromosomes;
+
+ /** maximal size of the population */
+ private int populationLimit;
+
+ /**
+ * Creates a new ListPopulation instance and initializes its inner chromosome list.
+ *
+ * @param populationLimit maximal size of the population
+ * @throws NotPositiveException if the population limit is not a positive number (< 1)
+ */
+ public ListPopulation(final int populationLimit) throws NotPositiveException {
+ this(Collections.<Chromosome> emptyList(), populationLimit);
+ }
+
+ /**
+ * Creates a new ListPopulation instance.
+ * <p>
+ * Note: the chromosomes of the specified list are added to the population.
+ *
+ * @param chromosomes list of chromosomes to be added to the population
+ * @param populationLimit maximal size of the population
+ * @throws NullArgumentException if the list of chromosomes is {@code null}
+ * @throws NotPositiveException if the population limit is not a positive number (< 1)
+ * @throws NumberIsTooLargeException if the list of chromosomes exceeds the population limit
+ */
+ public ListPopulation(final List<Chromosome> chromosomes, final int populationLimit)
+ throws NullArgumentException, NotPositiveException, NumberIsTooLargeException {
+
+ if (chromosomes == null) {
+ throw new NullArgumentException();
+ }
+ if (populationLimit <= 0) {
+ throw new NotPositiveException(LocalizedFormats.POPULATION_LIMIT_NOT_POSITIVE, populationLimit);
+ }
+ if (chromosomes.size() > populationLimit) {
+ throw new NumberIsTooLargeException(LocalizedFormats.LIST_OF_CHROMOSOMES_BIGGER_THAN_POPULATION_SIZE,
+ chromosomes.size(), populationLimit, false);
+ }
+ this.populationLimit = populationLimit;
+ this.chromosomes = new ArrayList<Chromosome>(populationLimit);
+ this.chromosomes.addAll(chromosomes);
+ }
+
+ /**
+ * Add a {@link Collection} of chromosomes to this {@link Population}.
+ * @param chromosomeColl a {@link Collection} of chromosomes
+ * @throws NumberIsTooLargeException if the population would exceed the population limit when
+ * adding this chromosome
+ * @since 3.1
+ */
+ public void addChromosomes(final Collection<Chromosome> chromosomeColl) throws NumberIsTooLargeException {
+ if (chromosomes.size() + chromosomeColl.size() > populationLimit) {
+ throw new NumberIsTooLargeException(LocalizedFormats.LIST_OF_CHROMOSOMES_BIGGER_THAN_POPULATION_SIZE,
+ chromosomes.size(), populationLimit, false);
+ }
+ this.chromosomes.addAll(chromosomeColl);
+ }
+
+ /**
+ * Returns an unmodifiable list of the chromosomes in this population.
+ * @return the unmodifiable list of chromosomes
+ */
+ public List<Chromosome> getChromosomes() {
+ return Collections.unmodifiableList(chromosomes);
+ }
+
+ /**
+ * Sets the list of chromosomes.
+ * <p>
+ * Note: this method removes all existing chromosomes in the population and adds all chromosomes
+ * of the specified list to the population.
+ *
+ * @param chromosomes the list of chromosomes
+ * @throws NullArgumentException if the list of chromosomes is {@code null}
+ * @throws NumberIsTooLargeException if the list of chromosomes exceeds the population limit
+ * @deprecated use {@link #addChromosomes(Collection)} instead
+ */
+ @Deprecated
+ public void setChromosomes(final List<Chromosome> chromosomes)
+ throws NullArgumentException, NumberIsTooLargeException {
+
+ if (chromosomes == null) {
+ throw new NullArgumentException();
+ }
+ if (chromosomes.size() > populationLimit) {
+ throw new NumberIsTooLargeException(LocalizedFormats.LIST_OF_CHROMOSOMES_BIGGER_THAN_POPULATION_SIZE,
+ chromosomes.size(), populationLimit, false);
+ }
+ this.chromosomes.clear();
+ this.chromosomes.addAll(chromosomes);
+ }
+
+ /**
+ * Access the list of chromosomes.
+ * @return the list of chromosomes
+ * @since 3.1
+ */
+ protected List<Chromosome> getChromosomeList() {
+ return chromosomes;
+ }
+
+ /**
+ * Add the given chromosome to the population.
+ *
+ * @param chromosome the chromosome to add.
+ * @throws NumberIsTooLargeException if the population would exceed the {@code populationLimit} after
+ * adding this chromosome
+ */
+ public void addChromosome(final Chromosome chromosome) throws NumberIsTooLargeException {
+ if (chromosomes.size() >= populationLimit) {
+ throw new NumberIsTooLargeException(LocalizedFormats.LIST_OF_CHROMOSOMES_BIGGER_THAN_POPULATION_SIZE,
+ chromosomes.size(), populationLimit, false);
+ }
+ this.chromosomes.add(chromosome);
+ }
+
+ /**
+ * Access the fittest chromosome in this population.
+ * @return the fittest chromosome.
+ */
+ public Chromosome getFittestChromosome() {
+ // best so far
+ Chromosome bestChromosome = this.chromosomes.get(0);
+ for (Chromosome chromosome : this.chromosomes) {
+ if (chromosome.compareTo(bestChromosome) > 0) {
+ // better chromosome found
+ bestChromosome = chromosome;
+ }
+ }
+ return bestChromosome;
+ }
+
+ /**
+ * Access the maximum population size.
+ * @return the maximum population size.
+ */
+ public int getPopulationLimit() {
+ return this.populationLimit;
+ }
+
+ /**
+ * Sets the maximal population size.
+ * @param populationLimit maximal population size.
+ * @throws NotPositiveException if the population limit is not a positive number (< 1)
+ * @throws NumberIsTooSmallException if the new population size is smaller than the current number
+ * of chromosomes in the population
+ */
+ public void setPopulationLimit(final int populationLimit) throws NotPositiveException, NumberIsTooSmallException {
+ if (populationLimit <= 0) {
+ throw new NotPositiveException(LocalizedFormats.POPULATION_LIMIT_NOT_POSITIVE, populationLimit);
+ }
+ if (populationLimit < chromosomes.size()) {
+ throw new NumberIsTooSmallException(populationLimit, chromosomes.size(), true);
+ }
+ this.populationLimit = populationLimit;
+ }
+
+ /**
+ * Access the current population size.
+ * @return the current population size.
+ */
+ public int getPopulationSize() {
+ return this.chromosomes.size();
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public String toString() {
+ return this.chromosomes.toString();
+ }
+
+ /**
+ * Returns an iterator over the unmodifiable list of chromosomes.
+ * <p>Any call to {@link Iterator#remove()} will result in a {@link UnsupportedOperationException}.</p>
+ *
+ * @return chromosome iterator
+ */
+ public Iterator<Chromosome> iterator() {
+ return getChromosomes().iterator();
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/7b6606ad/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/lib/MutationPolicy.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/lib/MutationPolicy.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/lib/MutationPolicy.java
new file mode 100755
index 0000000..99ba876
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/lib/MutationPolicy.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.cube.cuboid.algorithm.generic.lib;
+
+/**
+ * Algorithm used to mutate a chromosome.
+ *
+ */
+public interface MutationPolicy {
+
+ /**
+ * Mutate the given chromosome.
+ * @param original the original chromosome.
+ * @return the mutated chromosome.
+ * @throws IllegalArgumentException if the given chromosome is not compatible with this {@link MutationPolicy}
+ */
+ Chromosome mutate(Chromosome original) throws IllegalArgumentException;
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/7b6606ad/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/lib/OnePointCrossover.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/lib/OnePointCrossover.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/lib/OnePointCrossover.java
new file mode 100755
index 0000000..1f484cb
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/lib/OnePointCrossover.java
@@ -0,0 +1,126 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.cube.cuboid.algorithm.generic.lib;
+
+import java.util.BitSet;
+
+import org.apache.kylin.cube.cuboid.algorithm.generic.BitsChromosome;
+import org.apache.kylin.cube.cuboid.algorithm.generic.GeneticAlgorithm;
+
+/**
+ * One point crossover policy. A random crossover point is selected and the
+ * first part from each parent is copied to the corresponding child, and the
+ * second parts are copied crosswise.
+ *
+ * Example:
+ * <pre>
+ * -C- denotes a crossover point
+ * -C- -C-
+ * p1 = (1 0 1 0 0 1 | 0 1 1) X p2 = (0 1 1 0 1 0 | 1 1 1)
+ * \------------/ \-----/ \------------/ \-----/
+ * || (*) || (**)
+ * VV (**) VV (*)
+ * /------------\ /-----\ /------------\ /-----\
+ * c1 = (1 0 1 0 0 1 | 1 1 1) X c2 = (0 1 1 0 1 0 | 0 1 1)
+ * </pre>
+ *
+ * This policy works only on {@link BitsChromosome}, and therefore it
+ * is parameterized by T. Moreover, the chromosomes must have same lengths.
+ *
+ *
+ */
+public class OnePointCrossover implements CrossoverPolicy {
+
+ /**
+ * Performs one point crossover. A random crossover point is selected and the
+ * first part from each parent is copied to the corresponding child, and the
+ * second parts are copied crosswise.
+ *
+ * Example:
+ * <pre>
+ * -C- denotes a crossover point
+ * -C- -C-
+ * p1 = (1 0 1 0 0 1 | 0 1 1) X p2 = (0 1 1 0 1 0 | 1 1 1)
+ * \------------/ \-----/ \------------/ \-----/
+ * || (*) || (**)
+ * VV (**) VV (*)
+ * /------------\ /-----\ /------------\ /-----\
+ * c1 = (1 0 1 0 0 1 | 1 1 1) X c2 = (0 1 1 0 1 0 | 0 1 1)
+ * </pre>
+ *
+ * @param first first parent (p1)
+ * @param second second parent (p2)
+ * @return pair of two children (c1,c2)
+ * @throws IllegalArgumentException if one of the chromosomes is
+ * not an instance of {@link BitsChromosome}
+ * @throws ChromosomeMismatchException if the length of the two chromosomes is different
+ */
+ @SuppressWarnings("unchecked") // OK because of instanceof checks
+ public ChromosomePair crossover(final Chromosome first, final Chromosome second)
+ throws IllegalArgumentException, ChromosomeMismatchException {
+
+ if (!(first instanceof BitsChromosome && second instanceof BitsChromosome)) {
+ throw new IllegalArgumentException("Chromosome first " + first.getClass() + " and second "
+ + second.getClass() + " must be of type BitsChromosome.");
+ }
+ return crossover((BitsChromosome) first, (BitsChromosome) second);
+ }
+
+ /**
+ * Helper for {@link #crossover(Chromosome, Chromosome)}. Performs the actual crossover.
+ *
+ * @param first the first chromosome.
+ * @param second the second chromosome.
+ * @return the pair of new chromosomes that resulted from the crossover.
+ * @throws ChromosomeMismatchException if the length of the two chromosomes is different
+ */
+ private ChromosomePair crossover(final BitsChromosome first, final BitsChromosome second)
+ throws ChromosomeMismatchException {
+ final int length = first.getLength();
+ if (length != second.getLength()) {
+ throw new ChromosomeMismatchException("BitsChromosome length mismatch.", second.getLength(), length);
+ }
+
+ final BitSet parent1Key = first.getKey();
+ final BitSet parent2Key = second.getKey();
+
+ final BitSet child1Key = new BitSet(length);
+ final BitSet child2Key = new BitSet(length);
+
+ // select a crossover point at random (0 and length makes no sense)
+ final int crossoverIndex = 1 + (GeneticAlgorithm.RANDGEN.get().nextInt(length - 2));
+
+ BitSet a = (BitSet) parent1Key.clone();
+ a.clear(crossoverIndex, length);
+ BitSet b = (BitSet) parent2Key.clone();
+ b.clear(0, crossoverIndex);
+
+ BitSet c = (BitSet) parent1Key.clone();
+ c.clear(crossoverIndex, length);
+ BitSet d = (BitSet) parent2Key.clone();
+ d.clear(0, crossoverIndex);
+
+ child1Key.or(a);
+ child1Key.or(d);
+
+ child2Key.or(c);
+ child2Key.or(b);
+ return new ChromosomePair(first.newBitsChromosome(child1Key), second.newBitsChromosome(child2Key));
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/7b6606ad/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/lib/Population.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/lib/Population.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/lib/Population.java
new file mode 100755
index 0000000..8e35420
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/lib/Population.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.cube.cuboid.algorithm.generic.lib;
+
+import org.apache.commons.math3.exception.NumberIsTooLargeException;
+
+/**
+ * A collection of chromosomes that facilitates generational evolution.
+ *
+ */
+public interface Population extends Iterable<Chromosome> {
+ /**
+ * Access the current population size.
+ * @return the current population size.
+ */
+ int getPopulationSize();
+
+ /**
+ * Access the maximum population size.
+ * @return the maximum population size.
+ */
+ int getPopulationLimit();
+
+ /**
+ * Start the population for the next generation.
+ * @return the beginnings of the next generation.
+ */
+ Population nextGeneration();
+
+ /**
+ * Add the given chromosome to the population.
+ * @param chromosome the chromosome to add.
+ * @throws NumberIsTooLargeException if the population would exceed the population limit when adding
+ * this chromosome
+ */
+ void addChromosome(Chromosome chromosome) throws NumberIsTooLargeException;
+
+ /**
+ * Access the fittest chromosome in this population.
+ * @return the fittest chromosome.
+ */
+ Chromosome getFittestChromosome();
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/7b6606ad/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/lib/SelectionPolicy.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/lib/SelectionPolicy.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/lib/SelectionPolicy.java
new file mode 100755
index 0000000..1c32f88
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/lib/SelectionPolicy.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.cube.cuboid.algorithm.generic.lib;
+
+/**
+ * Algorithm used to select a chromosome pair from a population.
+ *
+ */
+public interface SelectionPolicy {
+ /**
+ * Select two chromosomes from the population.
+ * @param population the population from which the chromosomes are choosen.
+ * @return the selected chromosomes.
+ * @throws IllegalArgumentException if the population is not compatible with this {@link SelectionPolicy}
+ */
+ ChromosomePair select(Population population) throws IllegalArgumentException;
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/7b6606ad/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/lib/StoppingCondition.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/lib/StoppingCondition.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/lib/StoppingCondition.java
new file mode 100755
index 0000000..28b879a
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/lib/StoppingCondition.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.cube.cuboid.algorithm.generic.lib;
+
+/**
+ * Algorithm used to determine when to stop evolution.
+ *
+ */
+public interface StoppingCondition {
+ /**
+ * Determine whether or not the given population satisfies the stopping condition.
+ *
+ * @param population the population to test.
+ * @return <code>true</code> if this stopping condition is met by the given population,
+ * <code>false</code> otherwise.
+ */
+ boolean isSatisfied(Population population);
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/7b6606ad/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/lib/TournamentSelection.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/lib/TournamentSelection.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/lib/TournamentSelection.java
new file mode 100755
index 0000000..cca44ca
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/generic/lib/TournamentSelection.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.cube.cuboid.algorithm.generic.lib;
+
+import java.util.List;
+
+import org.apache.kylin.cube.cuboid.algorithm.generic.GeneticAlgorithm;
+
+import com.google.common.collect.Lists;
+
+/**
+ * Tournament selection scheme. Each of the two selected chromosomes is selected
+ * based on n-ary tournament -- this is done by drawing {@link #arity} random
+ * chromosomes without replacement from the population, and then selecting the
+ * fittest chromosome among them.
+ *
+ */
+public class TournamentSelection implements SelectionPolicy {
+
+ /** number of chromosomes included in the tournament selections */
+ private int arity;
+
+ /**
+ * Creates a new TournamentSelection instance.
+ *
+ * @param arity how many chromosomes will be drawn to the tournament
+ */
+ public TournamentSelection(final int arity) {
+ this.arity = arity;
+ }
+
+ /**
+ * Select two chromosomes from the population. Each of the two selected
+ * chromosomes is selected based on n-ary tournament -- this is done by
+ * drawing {@link #arity} random chromosomes without replacement from the
+ * population, and then selecting the fittest chromosome among them.
+ *
+ * @param population the population from which the chromosomes are chosen.
+ * @return the selected chromosomes.
+ * @throws IllegalArgumentException if the tournament arity is bigger than the population size
+ */
+ public ChromosomePair select(final Population population) throws IllegalArgumentException {
+ return new ChromosomePair(tournament((ListPopulation) population), tournament((ListPopulation) population));
+ }
+
+ /**
+ * Helper for {@link #select(Population)}. Draw {@link #arity} random chromosomes without replacement from the
+ * population, and then select the fittest chromosome among them.
+ *
+ * @param population the population from which the chromosomes are chosen.
+ * @return the selected chromosome.
+ * @throws IllegalArgumentException if the tournament arity is bigger than the population size
+ */
+ private Chromosome tournament(final ListPopulation population) throws IllegalArgumentException {
+ if (population.getPopulationSize() < this.arity) {
+ throw new IllegalArgumentException("Tournament arty is too large.");
+ }
+ // auxiliary population
+ ListPopulation tournamentPopulation = new ListPopulation(this.arity) {
+ /** {@inheritDoc} */
+ public Population nextGeneration() {
+ // not useful here
+ return null;
+ }
+ };
+
+ // create a copy of the chromosome list
+ List<Chromosome> chromosomes = Lists.newArrayList(population.getChromosomes());
+ for (int i = 0; i < this.arity; i++) {
+ // select a random individual and add it to the tournament
+ int rind = GeneticAlgorithm.RANDGEN.get().nextInt(chromosomes.size());
+ tournamentPopulation.addChromosome(chromosomes.get(rind));
+ // do not select it again
+ chromosomes.remove(rind);
+ }
+ // the winner takes it all
+ return tournamentPopulation.getFittestChromosome();
+ }
+
+ /**
+ * Gets the arity (number of chromosomes drawn to the tournament).
+ *
+ * @return arity of the tournament
+ */
+ public int getArity() {
+ return arity;
+ }
+
+ /**
+ * Sets the arity (number of chromosomes drawn to the tournament).
+ *
+ * @param arity arity of the tournament
+ */
+ public void setArity(final int arity) {
+ this.arity = arity;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/7b6606ad/core-cube/src/test/java/org/apache/kylin/cube/cuboid/algorithm/GeneticAlgorithmTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/cuboid/algorithm/GeneticAlgorithmTest.java b/core-cube/src/test/java/org/apache/kylin/cube/cuboid/algorithm/GeneticAlgorithmTest.java
new file mode 100755
index 0000000..4cefbd4
--- /dev/null
+++ b/core-cube/src/test/java/org/apache/kylin/cube/cuboid/algorithm/GeneticAlgorithmTest.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.cube.cuboid.algorithm;
+
+import java.util.List;
+
+import org.apache.kylin.cube.cuboid.algorithm.generic.GeneticAlgorithm;
+import org.junit.Test;
+
+public class GeneticAlgorithmTest extends AlgorithmTestBase {
+
+ @Test
+ public void testBPUSCalculator() {
+ BenefitPolicy benefitPolicy = new BPUSCalculator(cuboidStats);
+ GeneticAlgorithm algorithm = new GeneticAlgorithm(-1, benefitPolicy, cuboidStats);
+
+ List<Long> recommendList = algorithm.recommend(10);
+ System.out.println("recommendList by BPUSCalculator: " + recommendList);
+ System.out.println("Cost evaluated for each query: " + getQueryCostRatio(cuboidStats, recommendList));
+ }
+
+ @Test
+ public void testPBPUSCalculator() {
+ BenefitPolicy benefitPolicy = new PBPUSCalculator(cuboidStats);
+ GeneticAlgorithm algorithm = new GeneticAlgorithm(-1, benefitPolicy, cuboidStats);
+
+ List<Long> recommendList = algorithm.recommend(10);
+ System.out.println("recommendList by PBPUSCalculator:" + recommendList);
+ System.out.println("Cost evaluated for each query: " + getQueryCostRatio(cuboidStats, recommendList));
+ }
+
+ @Test
+ public void testSPBPUSCalculator() {
+ BenefitPolicy benefitPolicy = new SPBPUSCalculator(cuboidStats);
+ GeneticAlgorithm algorithm = new GeneticAlgorithm(-1, benefitPolicy, cuboidStats);
+
+ List<Long> recommendList = algorithm.recommend(10);
+ System.out.println("recommendList by SPBPUSCalculator:" + recommendList);
+ System.out.println("Cost evaluated for each query: " + getQueryCostRatio(cuboidStats, recommendList));
+ }
+}
[02/13] kylin git commit: KYLIN-2877 fix unclosed prepared statement
Posted by li...@apache.org.
KYLIN-2877 fix unclosed prepared statement
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ae430ed8
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ae430ed8
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ae430ed8
Branch: refs/heads/pr70
Commit: ae430ed8745ea36679113f27d745f6cca7898ac0
Parents: ff14042
Author: Li Yang <li...@apache.org>
Authored: Sat Sep 23 08:30:17 2017 +0800
Committer: Li Yang <li...@apache.org>
Committed: Sat Sep 23 08:30:17 2017 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/rest/service/QueryService.java | 9 +++++----
1 file changed, 5 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/ae430ed8/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
index 8a3e6b3..a858fbd 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
@@ -811,12 +811,13 @@ public class QueryService extends BasicService {
if (isPrepareStatementWithParams(sqlRequest)) {
- PreparedStatement preparedState = conn.prepareStatement(correctedSql);
- processStatementAttr(preparedState, sqlRequest);
+ stat = conn.prepareStatement(correctedSql); // to be closed in the finally
+ PreparedStatement prepared = (PreparedStatement) stat;
+ processStatementAttr(prepared, sqlRequest);
for (int i = 0; i < ((PrepareSqlRequest) sqlRequest).getParams().length; i++) {
- setParam(preparedState, i + 1, ((PrepareSqlRequest) sqlRequest).getParams()[i]);
+ setParam(prepared, i + 1, ((PrepareSqlRequest) sqlRequest).getParams()[i]);
}
- resultSet = preparedState.executeQuery();
+ resultSet = prepared.executeQuery();
} else {
stat = conn.createStatement();
processStatementAttr(stat, sqlRequest);
[05/13] kylin git commit: KYLIN-2718 use Guava Library to check
overflow when calculating combination
Posted by li...@apache.org.
KYLIN-2718 use Guava Library to check overflow when calculating combination
Signed-off-by: Li Yang <li...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/228b34aa
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/228b34aa
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/228b34aa
Branch: refs/heads/pr70
Commit: 228b34aa10f3f89f9024bc2368cbe11a59894d28
Parents: 713e9f7
Author: Zhong <nj...@apache.org>
Authored: Sat Sep 23 15:11:49 2017 +0800
Committer: Li Yang <li...@apache.org>
Committed: Sat Sep 23 15:12:09 2017 +0800
----------------------------------------------------------------------
.../kylin/cube/model/AggregationGroup.java | 73 +++++++++++---------
.../org/apache/kylin/cube/model/CubeDesc.java | 2 -
.../cube/model/TooManyCuboidException.java | 1 +
3 files changed, 41 insertions(+), 35 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/228b34aa/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java b/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java
index d473858..74ac715 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/AggregationGroup.java
@@ -41,6 +41,7 @@ import com.google.common.base.Preconditions;
import com.google.common.collect.Collections2;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
+import com.google.common.math.LongMath;
@SuppressWarnings("serial")
@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
@@ -300,42 +301,48 @@ public class AggregationGroup implements Serializable {
public long calculateCuboidCombination() {
long combination = 1;
- if (this.getDimCap() > 0) {
- CuboidScheduler cuboidScheduler = cubeDesc.getInitialCuboidScheduler();
- combination = cuboidScheduler.calculateCuboidsForAggGroup(this).size();
- } else {
- Set<String> includeDims = new TreeSet<>(Arrays.asList(includes));
- Set<String> mandatoryDims = new TreeSet<>(Arrays.asList(selectRule.mandatoryDims));
-
- Set<String> hierarchyDims = new TreeSet<>();
- for (String[] ss : selectRule.hierarchyDims) {
- hierarchyDims.addAll(Arrays.asList(ss));
- combination = combination * (ss.length + 1);
- }
-
- Set<String> jointDims = new TreeSet<>();
- for (String[] ss : selectRule.jointDims) {
- jointDims.addAll(Arrays.asList(ss));
- combination = combination * 2;
- }
-
- Set<String> normalDims = new TreeSet<>();
- normalDims.addAll(includeDims);
- normalDims.removeAll(mandatoryDims);
- normalDims.removeAll(hierarchyDims);
- normalDims.removeAll(jointDims);
-
- combination = combination * (1L << normalDims.size());
-
- if (cubeDesc.getConfig().getCubeAggrGroupIsMandatoryOnlyValid() && !mandatoryDims.isEmpty()) {
- combination += 1;
+ try {
+ if (this.getDimCap() > 0) {
+ CuboidScheduler cuboidScheduler = cubeDesc.getInitialCuboidScheduler();
+ combination = cuboidScheduler.calculateCuboidsForAggGroup(this).size();
+ } else {
+ Set<String> includeDims = new TreeSet<>(Arrays.asList(includes));
+ Set<String> mandatoryDims = new TreeSet<>(Arrays.asList(selectRule.mandatoryDims));
+
+ Set<String> hierarchyDims = new TreeSet<>();
+ for (String[] ss : selectRule.hierarchyDims) {
+ hierarchyDims.addAll(Arrays.asList(ss));
+ combination = LongMath.checkedMultiply(combination, (ss.length + 1));
+ }
+
+ Set<String> jointDims = new TreeSet<>();
+ for (String[] ss : selectRule.jointDims) {
+ jointDims.addAll(Arrays.asList(ss));
+ }
+ combination = LongMath.checkedMultiply(combination, (1L << selectRule.jointDims.length));
+
+ Set<String> normalDims = new TreeSet<>();
+ normalDims.addAll(includeDims);
+ normalDims.removeAll(mandatoryDims);
+ normalDims.removeAll(hierarchyDims);
+ normalDims.removeAll(jointDims);
+
+ combination = LongMath.checkedMultiply(combination, (1L << normalDims.size()));
+
+ if (cubeDesc.getConfig().getCubeAggrGroupIsMandatoryOnlyValid() && !mandatoryDims.isEmpty()) {
+ combination += 1;
+ }
+ combination -= 1; // not include cuboid 0
}
- combination -= 1; // not include cuboid 0
+
+ if (combination < 0)
+ throw new ArithmeticException();
+
+ } catch (ArithmeticException ae) {
+ // long overflow, give max value
+ combination = Long.MAX_VALUE;
}
- if (combination < 0) { // overflow
- combination = Long.MAX_VALUE - 1;
- }
return combination;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/228b34aa/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
index 2a902f3..712af43 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
@@ -660,8 +660,6 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware {
+ ". Use 'mandatory'/'hierarchy'/'joint' to optimize; or update 'kylin.cube.aggrgroup.max-combination' to a bigger value.";
throw new TooManyCuboidException(msg);
}
- } catch (TooManyCuboidException tmce) {
- throw tmce;
} catch (Exception e) {
throw new IllegalStateException("Unknown error while calculating cuboid number for " + //
"Aggregation group " + index + " of Cube Desc " + this.name, e);
http://git-wip-us.apache.org/repos/asf/kylin/blob/228b34aa/core-cube/src/main/java/org/apache/kylin/cube/model/TooManyCuboidException.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/TooManyCuboidException.java b/core-cube/src/main/java/org/apache/kylin/cube/model/TooManyCuboidException.java
index cfa41e4..57ea6ed 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/TooManyCuboidException.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/TooManyCuboidException.java
@@ -17,6 +17,7 @@
*/
package org.apache.kylin.cube.model;
+@SuppressWarnings("serial")
public class TooManyCuboidException extends RuntimeException {
public TooManyCuboidException(String message) {
[06/13] kylin git commit: KYLIN-2622 Add SegmentAppendTrieDictBuilder
for count distinct measure
Posted by li...@apache.org.
KYLIN-2622 Add SegmentAppendTrieDictBuilder for count distinct measure
Signed-off-by: Li Yang <li...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7530edb5
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7530edb5
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7530edb5
Branch: refs/heads/pr70
Commit: 7530edb507b7f174e39413c24ed594d587c4fe99
Parents: 228b34a
Author: kangkaisen <ka...@live.com>
Authored: Tue May 16 17:04:55 2017 +0800
Committer: Li Yang <li...@apache.org>
Committed: Sat Sep 23 16:28:58 2017 +0800
----------------------------------------------------------------------
.../apache/kylin/dict/AppendTrieDictionary.java | 9 ++-
.../apache/kylin/dict/DictionaryManager.java | 10 +++
.../kylin/dict/GlobalDictionaryBuilder.java | 8 +--
.../global/AppendTrieDictionaryBuilder.java | 15 ++--
.../kylin/dict/global/GlobalDictHDFSStore.java | 54 +++++++++-----
.../kylin/dict/global/GlobalDictMetadata.java | 1 -
.../kylin/dict/global/GlobalDictStore.java | 5 +-
.../global/SegmentAppendTrieDictBuilder.java | 76 ++++++++++++++++++++
.../dict/global/AppendTrieDictionaryTest.java | 3 +-
.../kylin/measure/bitmap/BitmapMeasureType.java | 21 +-----
.../localmeta/cube_desc/ci_inner_join_cube.json | 2 +-
webapp/app/js/model/cubeConfig.js | 3 +-
.../cubeDesigner/advanced_settings.html | 13 +++-
13 files changed, 162 insertions(+), 58 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/7530edb5/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java b/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java
index b9f0d2b..18795b4 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java
@@ -38,8 +38,8 @@ import java.io.IOException;
import java.io.PrintStream;
import java.util.Arrays;
import java.util.Objects;
+import java.util.TreeMap;
import java.util.concurrent.ExecutionException;
-import static com.google.common.base.Preconditions.checkState;
/**
* A dictionary based on Trie data structure that maps enumerations of byte[] to
@@ -72,7 +72,12 @@ public class AppendTrieDictionary<T> extends CacheDictionary<T> {
this.baseDir = baseDir;
final GlobalDictStore globalDictStore = new GlobalDictHDFSStore(baseDir);
Long[] versions = globalDictStore.listAllVersions();
- checkState(versions.length > 0, "Global dict at %s is empty", baseDir);
+
+ if (versions.length == 0) {
+ this.metadata = new GlobalDictMetadata(0, 0, 0, 0, null, new TreeMap<AppendDictSliceKey, String>());
+ return; // for the removed SegmentAppendTrieDictBuilder
+ }
+
final long latestVersion = versions[versions.length - 1];
final Path latestVersionPath = globalDictStore.getVersionDir(latestVersion);
this.metadata = globalDictStore.getMetadata(latestVersion);
http://git-wip-us.apache.org/repos/asf/kylin/blob/7530edb5/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
index 135ab0c..0eab8e4 100755
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
@@ -218,6 +218,16 @@ public class DictionaryManager {
if (dicts.size() == 1)
return dicts.get(0);
+ /**
+ * AppendTrieDictionary needn't merge
+ * more than one AppendTrieDictionary will generate when user use {@link SegmentAppendTrieDictBuilder}
+ */
+ for (DictionaryInfo dict: dicts) {
+ if (dict.getDictionaryClass().equals(AppendTrieDictionary.class.getName())) {
+ return dict;
+ }
+ }
+
DictionaryInfo firstDictInfo = null;
int totalSize = 0;
for (DictionaryInfo info : dicts) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/7530edb5/core-dictionary/src/main/java/org/apache/kylin/dict/GlobalDictionaryBuilder.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/GlobalDictionaryBuilder.java b/core-dictionary/src/main/java/org/apache/kylin/dict/GlobalDictionaryBuilder.java
index a593371..404d53c 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/GlobalDictionaryBuilder.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/GlobalDictionaryBuilder.java
@@ -44,16 +44,13 @@ public class GlobalDictionaryBuilder implements IDictionaryBuilder {
@Override
public void init(DictionaryInfo dictInfo, int baseId) throws IOException {
- if (dictInfo == null) {
- throw new IllegalArgumentException("GlobalDictinaryBuilder must used with an existing DictionaryInfo");
- }
-
sourceColumn = dictInfo.getSourceTable() + "_" + dictInfo.getSourceColumn();
lock = KylinConfig.getInstanceFromEnv().getDistributedLockFactory().lockForCurrentThread();
lock.lock(getLockPath(sourceColumn), Long.MAX_VALUE);
int maxEntriesPerSlice = KylinConfig.getInstanceFromEnv().getAppendDictEntrySize();
- this.builder = new AppendTrieDictionaryBuilder(dictInfo.getResourceDir(), maxEntriesPerSlice);
+ String baseDir = KylinConfig.getInstanceFromEnv().getHdfsWorkingDirectory() + "resources/GlobalDict" + dictInfo.getResourceDir() + "/";
+ this.builder = new AppendTrieDictionaryBuilder(baseDir, maxEntriesPerSlice, true);
this.baseId = baseId;
}
@@ -96,5 +93,4 @@ public class GlobalDictionaryBuilder implements IDictionaryBuilder {
private String getLockPath(String pathName) {
return "/dict/" + pathName + "/lock";
}
-
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/7530edb5/core-dictionary/src/main/java/org/apache/kylin/dict/global/AppendTrieDictionaryBuilder.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/global/AppendTrieDictionaryBuilder.java b/core-dictionary/src/main/java/org/apache/kylin/dict/global/AppendTrieDictionaryBuilder.java
index 90d65b6..54978c2 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/global/AppendTrieDictionaryBuilder.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/global/AppendTrieDictionaryBuilder.java
@@ -18,7 +18,6 @@
package org.apache.kylin.dict.global;
-import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.dict.AppendTrieDictionary;
import org.apache.kylin.dict.BytesConverter;
@@ -35,6 +34,7 @@ public class AppendTrieDictionaryBuilder {
private final String baseDir;
private final String workingDir;
private final int maxEntriesPerSlice;
+ private final boolean isAppendDictGlobal;
private GlobalDictStore store;
private int maxId;
@@ -46,20 +46,21 @@ public class AppendTrieDictionaryBuilder {
private AppendDictSliceKey curKey;
private AppendDictNode curNode;
- public AppendTrieDictionaryBuilder(String resourceDir, int maxEntriesPerSlice) throws IOException {
- this.baseDir = KylinConfig.getInstanceFromEnv().getHdfsWorkingDirectory() + "resources/GlobalDict" + resourceDir + "/";
- this.workingDir = this.baseDir + "/working";
+ public AppendTrieDictionaryBuilder(String baseDir, int maxEntriesPerSlice, boolean isAppendDictGlobal) throws IOException {
+ this.baseDir = baseDir;
+ this.workingDir = baseDir + "working";
this.maxEntriesPerSlice = maxEntriesPerSlice;
+ this.isAppendDictGlobal = isAppendDictGlobal;
init();
}
public synchronized void init() throws IOException {
this.store = new GlobalDictHDFSStore(baseDir);
- store.prepareForWrite(workingDir);
+ store.prepareForWrite(workingDir, isAppendDictGlobal);
Long[] versions = store.listAllVersions();
- if (versions.length == 0) { // build dict for the first time
+ if (versions.length == 0 || !isAppendDictGlobal) { // build dict for the first time
this.maxId = 0;
this.maxValueLength = 0;
this.nValues = 0;
@@ -122,7 +123,7 @@ public class AppendTrieDictionaryBuilder {
}
GlobalDictMetadata metadata = new GlobalDictMetadata(baseId, this.maxId, this.maxValueLength, this.nValues, this.bytesConverter, sliceFileMap);
- store.commit(workingDir, metadata);
+ store.commit(workingDir, metadata, isAppendDictGlobal);
AppendTrieDictionary dict = new AppendTrieDictionary();
dict.init(this.baseDir);
http://git-wip-us.apache.org/repos/asf/kylin/blob/7530edb5/core-dictionary/src/main/java/org/apache/kylin/dict/global/GlobalDictHDFSStore.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/global/GlobalDictHDFSStore.java b/core-dictionary/src/main/java/org/apache/kylin/dict/global/GlobalDictHDFSStore.java
index b30d5b9..4c8ce0f 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/global/GlobalDictHDFSStore.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/global/GlobalDictHDFSStore.java
@@ -59,13 +59,6 @@ public class GlobalDictHDFSStore extends GlobalDictStore {
this.basePath = new Path(baseDir);
this.conf = HadoopUtil.getCurrentConfiguration();
this.fileSystem = HadoopUtil.getFileSystem(baseDir);
-
- if (!fileSystem.exists(basePath)) {
- logger.info("Global dict at {} doesn't exist, create a new one", basePath);
- fileSystem.mkdirs(basePath);
- }
-
- migrateOldLayout();
}
// Previously we put slice files and index file directly in base directory,
@@ -111,8 +104,15 @@ public class GlobalDictHDFSStore extends GlobalDictStore {
}
@Override
- void prepareForWrite(String workingDir) throws IOException {
- // TODO create lock file
+ void prepareForWrite(String workingDir, boolean isGlobal) throws IOException {
+ if (!fileSystem.exists(basePath)) {
+ logger.info("Global dict at {} doesn't exist, create a new one", basePath);
+ fileSystem.mkdirs(basePath);
+ }
+
+ migrateOldLayout();
+
+ logger.info("Prepare to write Global dict at {}, isGlobal={}", workingDir, isGlobal);
Path working = new Path(workingDir);
if (fileSystem.exists(working)) {
@@ -122,7 +122,7 @@ public class GlobalDictHDFSStore extends GlobalDictStore {
// when build dict, copy all data into working dir and work on it, avoiding suddenly server crash made data corrupt
Long[] versions = listAllVersions();
- if (versions.length > 0) {
+ if (versions.length > 0 && isGlobal) {
Path latestVersion = getVersionDir(versions[versions.length - 1]);
FileUtil.copy(fileSystem, latestVersion, fileSystem, working, false, true, conf);
} else {
@@ -132,6 +132,10 @@ public class GlobalDictHDFSStore extends GlobalDictStore {
@Override
public Long[] listAllVersions() throws IOException {
+ if (!fileSystem.exists(basePath)) {
+ return new Long[0]; // for the removed SegmentAppendTrieDictBuilder
+ }
+
FileStatus[] versionDirs = fileSystem.listStatus(basePath, new PathFilter() {
@Override
public boolean accept(Path path) {
@@ -208,7 +212,7 @@ public class GlobalDictHDFSStore extends GlobalDictStore {
}
@Override
- public void commit(String workingDir, GlobalDictMetadata metadata) throws IOException {
+ public void commit(String workingDir, GlobalDictMetadata metadata, boolean isAppendDictGlobal) throws IOException {
Path workingPath = new Path(workingDir);
// delete v1 index file
@@ -225,22 +229,38 @@ public class GlobalDictHDFSStore extends GlobalDictStore {
Path newVersionPath = new Path(basePath, VERSION_PREFIX + System.currentTimeMillis());
fileSystem.rename(workingPath, newVersionPath);
- cleanUp();
+ cleanUp(isAppendDictGlobal);
}
// Check versions count, delete expired versions
- private void cleanUp() throws IOException {
- Long[] versions = listAllVersions();
+ private void cleanUp(boolean isAppendDictGlobal) throws IOException {
long timestamp = System.currentTimeMillis();
- for (int i = 0; i < versions.length - maxVersions; i++) {
- if (versions[i] + versionTTL < timestamp) {
- fileSystem.delete(getVersionDir(versions[i]), true);
+ if (isAppendDictGlobal) {
+ Long[] versions = listAllVersions();
+ for (int i = 0; i < versions.length - maxVersions; i++) {
+ if (versions[i] + versionTTL < timestamp) {
+ fileSystem.delete(getVersionDir(versions[i]), true);
+ }
+ }
+ } else {
+ FileStatus[] segmentDictDirs = fileSystem.listStatus(basePath.getParent());
+ for (FileStatus fileStatus : segmentDictDirs) {
+ String filePath = fileStatus.getPath().getName();
+ Long version = Long.parseLong(filePath.split("_")[1]);
+ if (version + versionTTL < timestamp) {
+ fileSystem.delete(new Path(basePath.getParent() + "/" + filePath), true);
+ }
}
}
}
@Override
public String copyToAnotherMeta(KylinConfig srcConfig, KylinConfig dstConfig) throws IOException {
+ if (baseDir.contains("resources/SegmentDict")) {
+ logger.info("SegmentAppendTrieDict needn't to copy");
+ return baseDir;
+ }
+
checkArgument(baseDir.startsWith(srcConfig.getHdfsWorkingDirectory()), "Please check why current directory {} doesn't belong to source working directory {}", baseDir, srcConfig.getHdfsWorkingDirectory());
final String dstBaseDir = baseDir.replaceFirst(srcConfig.getHdfsWorkingDirectory(), dstConfig.getHdfsWorkingDirectory());
http://git-wip-us.apache.org/repos/asf/kylin/blob/7530edb5/core-dictionary/src/main/java/org/apache/kylin/dict/global/GlobalDictMetadata.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/global/GlobalDictMetadata.java b/core-dictionary/src/main/java/org/apache/kylin/dict/global/GlobalDictMetadata.java
index 7c89ea2..7fe6b60 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/global/GlobalDictMetadata.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/global/GlobalDictMetadata.java
@@ -38,7 +38,6 @@ public class GlobalDictMetadata {
public GlobalDictMetadata(int baseId, int maxId, int maxValueLength, int nValues, BytesConverter bytesConverter, NavigableMap<AppendDictSliceKey, String> sliceFileMap) {
- Preconditions.checkNotNull(bytesConverter, "bytesConverter");
Preconditions.checkNotNull(sliceFileMap, "sliceFileMap");
this.baseId = baseId;
http://git-wip-us.apache.org/repos/asf/kylin/blob/7530edb5/core-dictionary/src/main/java/org/apache/kylin/dict/global/GlobalDictStore.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/global/GlobalDictStore.java b/core-dictionary/src/main/java/org/apache/kylin/dict/global/GlobalDictStore.java
index eaf0729..f686680 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/global/GlobalDictStore.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/global/GlobalDictStore.java
@@ -37,7 +37,7 @@ public abstract class GlobalDictStore {
}
// workingDir should be an absolute path, will create if not exists
- abstract void prepareForWrite(String workingDir) throws IOException;
+ abstract void prepareForWrite(String workingDir, boolean isGlobal) throws IOException;
/**
* @return all versions of this dictionary in ascending order
@@ -87,9 +87,10 @@ public abstract class GlobalDictStore {
* commit the <i>DictSlice</i> and <i>GlobalDictMetadata</i> in workingDir to new versionDir
* @param workingDir where store the tmp slice and index, should exist
* @param globalDictMetadata the metadata of global dict
+ * @param isAppendDictGlobal mark the append dict whether is global or not
* @throws IOException on I/O error
*/
- public abstract void commit(String workingDir, GlobalDictMetadata globalDictMetadata) throws IOException;
+ public abstract void commit(String workingDir, GlobalDictMetadata globalDictMetadata, boolean isAppendDictGlobal) throws IOException;
/**
* Copy the latest version of this dict to another meta. The source is unchanged.
http://git-wip-us.apache.org/repos/asf/kylin/blob/7530edb5/core-dictionary/src/main/java/org/apache/kylin/dict/global/SegmentAppendTrieDictBuilder.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/global/SegmentAppendTrieDictBuilder.java b/core-dictionary/src/main/java/org/apache/kylin/dict/global/SegmentAppendTrieDictBuilder.java
new file mode 100644
index 0000000..270deee
--- /dev/null
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/global/SegmentAppendTrieDictBuilder.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.dict.global;
+
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.Dictionary;
+import org.apache.kylin.dict.DictionaryInfo;
+import org.apache.kylin.dict.IDictionaryBuilder;
+
+import java.io.IOException;
+import java.util.UUID;
+
+/**
+ * SegmentAppendTrieDictBuilder based on one segment.
+ * SegmentAppendTrieDictBuilder only used for count distinct measure that needn't rollup among segments.
+ * SegmentAppendTrieDictBuilder could avoid AppendTrieDictionary infinite growth.
+ * SegmentAppendTrieDictBuilder doesn't support merge.
+ */
+public class SegmentAppendTrieDictBuilder implements IDictionaryBuilder {
+ private AppendTrieDictionaryBuilder builder;
+ private int baseId;
+ private String sourceColumn;
+
+ @Override
+ public void init(DictionaryInfo dictInfo, int baseId) throws IOException {
+ sourceColumn = dictInfo.getSourceTable() + "." + dictInfo.getSourceColumn();
+
+ int maxEntriesPerSlice = KylinConfig.getInstanceFromEnv().getAppendDictEntrySize();
+ if (hdfsDir == null) {
+ //build in Kylin job server
+ hdfsDir = KylinConfig.getInstanceFromEnv().getHdfsWorkingDirectory();
+ }
+ //use UUID to make each segment dict in different HDFS dir and support concurrent build
+ //use timestamp to make the segment dict easily to delete
+ String baseDir = KylinConfig.getInstanceFromEnv().getHdfsWorkingDirectory() + "resources/SegmentDict" + dictInfo.getResourceDir() + "/" + UUID.randomUUID().toString() + "_" + System.currentTimeMillis()+ "/";
+
+ this.builder = new AppendTrieDictionaryBuilder(baseDir, maxEntriesPerSlice, false);
+ this.baseId = baseId;
+ }
+
+ @Override
+ public boolean addValue(String value) {
+ if (value == null) {
+ return false;
+ }
+
+ try {
+ builder.addValue(value);
+ } catch (Throwable e) {
+ throw new RuntimeException(String.format("Failed to create global dictionary on %s ", sourceColumn), e);
+ }
+
+ return true;
+ }
+
+ @Override
+ public Dictionary<String> build() throws IOException {
+ return builder.build(baseId);
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/7530edb5/core-dictionary/src/test/java/org/apache/kylin/dict/global/AppendTrieDictionaryTest.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/test/java/org/apache/kylin/dict/global/AppendTrieDictionaryTest.java b/core-dictionary/src/test/java/org/apache/kylin/dict/global/AppendTrieDictionaryTest.java
index 47011fe..6b39c36 100644
--- a/core-dictionary/src/test/java/org/apache/kylin/dict/global/AppendTrieDictionaryTest.java
+++ b/core-dictionary/src/test/java/org/apache/kylin/dict/global/AppendTrieDictionaryTest.java
@@ -94,7 +94,8 @@ public class AppendTrieDictionaryTest extends LocalFileMetadataTestCase {
private AppendTrieDictionaryBuilder createBuilder(String resourceDir) throws IOException {
int maxEntriesPerSlice = KylinConfig.getInstanceFromEnv().getAppendDictEntrySize();
- return new AppendTrieDictionaryBuilder(resourceDir, maxEntriesPerSlice);
+ String baseDir = KylinConfig.getInstanceFromEnv().getHdfsWorkingDirectory() + "/resources/GlobalDict" + resourceDir + "/";
+ return new AppendTrieDictionaryBuilder(baseDir, maxEntriesPerSlice, true);
}
@Test
http://git-wip-us.apache.org/repos/asf/kylin/blob/7530edb5/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapMeasureType.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapMeasureType.java b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapMeasureType.java
index e4fb079..403d1b6 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapMeasureType.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapMeasureType.java
@@ -126,25 +126,8 @@ public class BitmapMeasureType extends MeasureType<BitmapCounter> {
@Override
public BitmapCounter reEncodeDictionary(BitmapCounter value, MeasureDesc measureDesc, Map<TblColRef, Dictionary<String>> oldDicts, Map<TblColRef, Dictionary<String>> newDicts) {
- if (!needDictionaryColumn(measureDesc.getFunction())) {
- return value;
- }
- TblColRef colRef = measureDesc.getFunction().getParameter().getColRefs().get(0);
- Dictionary<String> sourceDict = oldDicts.get(colRef);
- Dictionary<String> mergedDict = newDicts.get(colRef);
-
- BitmapCounter retValue = factory.newBitmap();
- for (int id : value) {
- int newId;
- String v = sourceDict.getValueFromId(id);
- if (v == null) {
- newId = mergedDict.nullId();
- } else {
- newId = mergedDict.getIdFromValue(v);
- }
- retValue.add(newId);
- }
- return retValue;
+ //BitmapCounter needn't reEncode
+ return value;
}
@Override
http://git-wip-us.apache.org/repos/asf/kylin/blob/7530edb5/examples/test_case_data/localmeta/cube_desc/ci_inner_join_cube.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube_desc/ci_inner_join_cube.json b/examples/test_case_data/localmeta/cube_desc/ci_inner_join_cube.json
index cfd8cd1..4346206 100644
--- a/examples/test_case_data/localmeta/cube_desc/ci_inner_join_cube.json
+++ b/examples/test_case_data/localmeta/cube_desc/ci_inner_join_cube.json
@@ -359,7 +359,7 @@
"dictionaries": [
{
"column": "TEST_KYLIN_FACT.TEST_COUNT_DISTINCT_BITMAP",
- "builder": "org.apache.kylin.dict.GlobalDictionaryBuilder"
+ "builder": "org.apache.kylin.dict.global.SegmentAppendTrieDictBuilder"
}
],
"rowkey": {
http://git-wip-us.apache.org/repos/asf/kylin/blob/7530edb5/webapp/app/js/model/cubeConfig.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/model/cubeConfig.js b/webapp/app/js/model/cubeConfig.js
index eec3f10..7caa19b 100644
--- a/webapp/app/js/model/cubeConfig.js
+++ b/webapp/app/js/model/cubeConfig.js
@@ -110,7 +110,8 @@ KylinApp.constant('cubeConfig', {
],
statusNeedNofity:['ERROR', 'DISCARDED', 'SUCCEED'],
buildDictionaries:[
- {name:"Global Dictionary", value:"org.apache.kylin.dict.GlobalDictionaryBuilder"}
+ {name:"Global Dictionary", value:"org.apache.kylin.dict.GlobalDictionaryBuilder"},
+ {name:"Segment Dictionary", value:"org.apache.kylin.dict.global.SegmentAppendTrieDictBuilder"}
],
needSetLengthEncodingList:['fixed_length','fixed_length_hex','int','integer']
});
http://git-wip-us.apache.org/repos/asf/kylin/blob/7530edb5/webapp/app/partials/cubeDesigner/advanced_settings.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/cubeDesigner/advanced_settings.html b/webapp/app/partials/cubeDesigner/advanced_settings.html
index 8062975..8768108 100755
--- a/webapp/app/partials/cubeDesigner/advanced_settings.html
+++ b/webapp/app/partials/cubeDesigner/advanced_settings.html
@@ -534,7 +534,18 @@
</script>
<script type="text/ng-template" id="AdvancedDictionariesTip.html">
<div>
- <h4>Special settings for dictionaries. Leave blank by default.</h4>
+ <h4>Special settings for dictionaries.</h4>
+ <ol>
+ <li>
+ "Global Dictionary" is the default dict for precise count distinct measure, which support rollup among all segments.
+ </li>
+ <li>
+ "Segment Dictionary" is the special dict for precise count distinct measure, which is based on one segment and could not
+ support rollup among segments.
+ Specifically, if your cube isn't partitioned or you can ensure all your SQLs will group by your partition_column,
+ you could use "Segment Dictionary" instead of "Global Dictionary".
+ </li>
+ </ol>
</div>
</script>
[09/13] kylin git commit: KYLIN-2729 Introduce greedy algorithm for
cube planner
Posted by li...@apache.org.
KYLIN-2729 Introduce greedy algorithm for cube planner
Signed-off-by: Li Yang <li...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/01fe425f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/01fe425f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/01fe425f
Branch: refs/heads/pr70
Commit: 01fe425f1f4998d213fb89197781f1933821d0b2
Parents: e2029dd
Author: Zhong <nj...@apache.org>
Authored: Tue Aug 22 14:58:18 2017 +0800
Committer: Li Yang <li...@apache.org>
Committed: Sat Sep 23 18:16:48 2017 +0800
----------------------------------------------------------------------
.../algorithm/greedy/GreedyAlgorithm.java | 170 +++++++++++++++++++
.../cuboid/algorithm/GreedyAlgorithmTest.java | 58 +++++++
2 files changed, 228 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/01fe425f/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/greedy/GreedyAlgorithm.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/greedy/GreedyAlgorithm.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/greedy/GreedyAlgorithm.java
new file mode 100755
index 0000000..0d04cc2
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/greedy/GreedyAlgorithm.java
@@ -0,0 +1,170 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.cube.cuboid.algorithm.greedy;
+
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.atomic.AtomicReference;
+
+import org.apache.kylin.cube.cuboid.algorithm.AbstractRecommendAlgorithm;
+import org.apache.kylin.cube.cuboid.algorithm.BenefitPolicy;
+import org.apache.kylin.cube.cuboid.algorithm.CuboidBenefitModel;
+import org.apache.kylin.cube.cuboid.algorithm.CuboidStats;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+
+/**
+* A simple implementation of the Greedy Algorithm , it chooses the cuboids which give
+* the greatest benefit based on expansion rate and time limitation.
+*/
+public class GreedyAlgorithm extends AbstractRecommendAlgorithm {
+ private static final Logger logger = LoggerFactory.getLogger(GreedyAlgorithm.class);
+
+ private static final int THREAD_NUM = 8;
+ private ExecutorService executor;
+
+ private Set<Long> selected = Sets.newLinkedHashSet();
+ private List<Long> remaining = Lists.newLinkedList();
+
+ public GreedyAlgorithm(final long timeout, BenefitPolicy benefitPolicy, CuboidStats cuboidStats) {
+ super(timeout, benefitPolicy, cuboidStats);
+ }
+
+ @Override
+ public List<Long> recommend(double expansionRate) {
+ double spaceLimit = getCuboidStats().getBaseCuboidSize() * expansionRate;
+ return start(spaceLimit);
+ }
+
+ @Override
+ public List<Long> start(double spaceLimit) {
+ logger.info("Greedy Algorithm started.");
+ executor = Executors.newFixedThreadPool(THREAD_NUM,
+ new ThreadFactoryBuilder().setNameFormat("greedy-algorithm-benefit-calculator-pool-%d").build());
+
+ getBenefitPolicy().initBeforeStart();
+
+ //Initial mandatory cuboids
+ selected.clear();
+ double remainingSpace = spaceLimit;
+ for (Long mandatoryOne : getCuboidStats().getAllCuboidsForMandatory()) {
+ selected.add(mandatoryOne);
+ if (getCuboidStats().getCuboidSize(mandatoryOne) != null) {
+ remainingSpace -= getCuboidStats().getCuboidSize(mandatoryOne);
+ }
+ }
+ //Initial remaining cuboid set
+ remaining.clear();
+ remaining.addAll(getCuboidStats().getAllCuboidsForSelection());
+
+ long round = 0;
+ while (true) {
+ if (shouldCancel()) {
+ break;
+ }
+ // Choose one cuboId having the maximum benefit per unit space in all available list
+ CuboidBenefitModel best = recommendBestOne();
+ // If return null, then we should finish the process and return
+ if (best == null) {
+ break;
+ }
+ // If we finally find the cuboid selected does not meet a minimum threshold of benefit (for
+ // example, a cuboid with 0.99M roll up from a parent cuboid with 1M
+ // rows), then we should finish the process and return
+ if (!getBenefitPolicy().ifEfficient(best)) {
+ break;
+ }
+
+ remainingSpace -= getCuboidStats().getCuboidSize(best.getCuboidId());
+ // If we finally find there is no remaining space, then we should finish the process and return
+ if (remainingSpace <= 0) {
+ break;
+ }
+ selected.add(best.getCuboidId());
+ remaining.remove(best.getCuboidId());
+ getBenefitPolicy().propagateAggregationCost(best.getCuboidId(), selected);
+ round++;
+ if (logger.isInfoEnabled()) {
+ logger.info(String.format("Recommend in round %d : %s", round, best.toString()));
+ }
+ }
+
+ executor.shutdown();
+
+ List<Long> excluded = Lists.newArrayList(remaining);
+ remaining.retainAll(selected);
+ Preconditions.checkArgument(remaining.size() == 0,
+ "There should be no intersection between excluded list and selected list.");
+ logger.info("Greedy Algorithm finished.");
+
+ if (logger.isInfoEnabled()) {
+ logger.info("Excluded cuboidId size:" + excluded.size());
+ logger.info("Excluded cuboidId detail:");
+ for (Long cuboid : excluded) {
+ logger.info(String.format("cuboidId %d and Cost: %d and Space: %f", cuboid,
+ getCuboidStats().getCuboidQueryCost(cuboid), getCuboidStats().getCuboidSize(cuboid)));
+ }
+ logger.info("Total Space:" + (spaceLimit - remainingSpace));
+ logger.info("Space Expansion Rate:" + (spaceLimit - remainingSpace) / getCuboidStats().getBaseCuboidSize());
+ }
+ return Lists.newArrayList(selected);
+ }
+
+ private CuboidBenefitModel recommendBestOne() {
+ final int selectedSize = selected.size();
+ final AtomicReference<CuboidBenefitModel> best = new AtomicReference<>();
+
+ final CountDownLatch counter = new CountDownLatch(remaining.size());
+ for (final Long cuboid : remaining) {
+ executor.submit(new Runnable() {
+ @Override
+ public void run() {
+ CuboidBenefitModel currentBest = best.get();
+ assert (selected.size() == selectedSize);
+ CuboidBenefitModel.BenefitModel benefitModel = getBenefitPolicy().calculateBenefit(cuboid, selected);
+ if (benefitModel != null && (currentBest == null || currentBest.getBenefit() == null
+ || benefitModel.getBenefit() > currentBest.getBenefit())) {
+ while (!best.compareAndSet(currentBest,
+ new CuboidBenefitModel(getCuboidStats().getCuboidModel(cuboid), benefitModel))) {
+ currentBest = best.get();
+ if (benefitModel.getBenefit() <= currentBest.getBenefit()) {
+ break;
+ }
+ }
+ }
+ counter.countDown();
+ }
+ });
+ }
+
+ try {
+ counter.await();
+ } catch (InterruptedException e) {
+ }
+ return best.get();
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/01fe425f/core-cube/src/test/java/org/apache/kylin/cube/cuboid/algorithm/GreedyAlgorithmTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/cuboid/algorithm/GreedyAlgorithmTest.java b/core-cube/src/test/java/org/apache/kylin/cube/cuboid/algorithm/GreedyAlgorithmTest.java
new file mode 100755
index 0000000..0c9fbae
--- /dev/null
+++ b/core-cube/src/test/java/org/apache/kylin/cube/cuboid/algorithm/GreedyAlgorithmTest.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.cube.cuboid.algorithm;
+
+import java.util.List;
+
+import org.apache.kylin.cube.cuboid.algorithm.greedy.GreedyAlgorithm;
+import org.junit.Test;
+
+public class GreedyAlgorithmTest extends AlgorithmTestBase {
+
+ @Test
+ public void testBPUSCalculator() {
+ BenefitPolicy benefitPolicy = new BPUSCalculator(cuboidStats);
+ GreedyAlgorithm algorithm = new GreedyAlgorithm(-1, benefitPolicy, cuboidStats);
+
+ List<Long> recommendList = algorithm.recommend(10);
+ System.out.println("recommendList by BPUSCalculator: " + recommendList);
+ System.out.println("Cost evaluated for each query: " + getQueryCostRatio(cuboidStats, recommendList));
+ }
+
+ @Test
+ public void testPBPUSCalculator() {
+ BenefitPolicy benefitPolicy = new PBPUSCalculator(cuboidStats);
+ GreedyAlgorithm algorithm = new GreedyAlgorithm(-1, benefitPolicy, cuboidStats);
+
+ List<Long> recommendList = algorithm.recommend(10);
+ System.out.println("recommendList by PBPUSCalculator:" + recommendList);
+ System.out.println("Cost evaluated for each query: " + getQueryCostRatio(cuboidStats, recommendList));
+ }
+
+ @Test
+ public void testSPBPUSCalculator() {
+ BenefitPolicy benefitPolicy = new SPBPUSCalculator(cuboidStats);
+ GreedyAlgorithm algorithm = new GreedyAlgorithm(-1, benefitPolicy, cuboidStats);
+
+ List<Long> recommendList = algorithm.recommend(10);
+ System.out.println("recommendList by SPBPUSCalculator:" + recommendList);
+ System.out.println("Cost evaluated for each query: " + getQueryCostRatio(cuboidStats, recommendList));
+ }
+
+}
[04/13] kylin git commit: minor,
fix apache RAT check on some test files
Posted by li...@apache.org.
minor, fix apache RAT check on some test files
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/713e9f74
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/713e9f74
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/713e9f74
Branch: refs/heads/pr70
Commit: 713e9f74bd5ae69def13edae14f26c57fbeee44c
Parents: 5c1a3d1
Author: Li Yang <li...@apache.org>
Authored: Sat Sep 23 06:02:41 2017 +0000
Committer: Li Yang <li...@apache.org>
Committed: Sat Sep 23 14:05:30 2017 +0800
----------------------------------------------------------------------
pom.xml | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/713e9f74/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index dca75bb..4993732 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1442,6 +1442,8 @@
<!--Test Data-->
<exclude>**/src/test/resources/**</exclude>
+ <exclude>examples/sample_cube/template/**</exclude>
+ <exclude>examples/test_case_data/localmeta/**</exclude>
<!-- generated files -->
<exclude>**/target/**</exclude>
@@ -1484,8 +1486,7 @@
<!-- MIT license -->
<exclude>webapp/app/css/AdminLTE.css</exclude>
<exclude>webapp/app/css/messenger-theme-ice.css</exclude>
- <exclude>webapp/app/js/directives/kylin_abn_tree_directive.js
- </exclude>
+ <exclude>webapp/app/js/directives/kylin_abn_tree_directive.js</exclude>
<exclude>webapp/app/js/directives/angular-tree-control.js</exclude>
<exclude>webapp/app/js/directives/datetimepicker.js</exclude>
<exclude>webapp/app/js/directives/select.js</exclude>
[13/13] kylin git commit: KYLIN-2826 add basic support classes for
cube planner algorithms
Posted by li...@apache.org.
KYLIN-2826 add basic support classes for cube planner algorithms
Signed-off-by: Li Yang <li...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e2029dd9
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e2029dd9
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e2029dd9
Branch: refs/heads/pr70
Commit: e2029dd9a6de405d5ab32f05f0a92e9f228abef3
Parents: 2edee49
Author: Zhong <nj...@apache.org>
Authored: Tue Aug 22 14:57:29 2017 +0800
Committer: Li Yang <li...@apache.org>
Committed: Sat Sep 23 18:16:48 2017 +0800
----------------------------------------------------------------------
.../kylin/cube/cuboid/TreeCuboidScheduler.java | 3 +-
.../algorithm/AbstractRecommendAlgorithm.java | 82 +
.../cube/cuboid/algorithm/BPUSCalculator.java | 165 +
.../cube/cuboid/algorithm/BenefitPolicy.java | 41 +
.../cuboid/algorithm/CuboidBenefitModel.java | 210 +
.../algorithm/CuboidRecommendAlgorithm.java | 49 +
.../cube/cuboid/algorithm/CuboidStats.java | 272 ++
.../cube/cuboid/algorithm/CuboidStatsUtil.java | 166 +
.../cube/cuboid/algorithm/PBPUSCalculator.java | 53 +
.../cube/cuboid/algorithm/SPBPUSCalculator.java | 41 +
.../cuboid/algorithm/AlgorithmTestBase.java | 310 ++
.../cuboid/algorithm/CuboidStatsUtilTest.java | 161 +
core-cube/src/test/resources/statistics.txt | 4092 ++++++++++++++++++
13 files changed, 5643 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/e2029dd9/core-cube/src/main/java/org/apache/kylin/cube/cuboid/TreeCuboidScheduler.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/TreeCuboidScheduler.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/TreeCuboidScheduler.java
index 8e3b4dc..903e358 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/TreeCuboidScheduler.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/TreeCuboidScheduler.java
@@ -89,8 +89,7 @@ public class TreeCuboidScheduler extends CuboidScheduler {
return createFromCuboids(allCuboidIds, Cuboid.cuboidSelectComparator);
}
- @VisibleForTesting
- static CuboidTree createFromCuboids(List<Long> allCuboidIds, Comparator<Long> cuboidComparator) {
+ public static CuboidTree createFromCuboids(List<Long> allCuboidIds, Comparator<Long> cuboidComparator) {
// sort the cuboid ids in descending order, so that don't need to adjust
// the cuboid tree when adding cuboid id to the tree.
Collections.sort(allCuboidIds, new Comparator<Long>() {
http://git-wip-us.apache.org/repos/asf/kylin/blob/e2029dd9/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/AbstractRecommendAlgorithm.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/AbstractRecommendAlgorithm.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/AbstractRecommendAlgorithm.java
new file mode 100755
index 0000000..b35c738
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/AbstractRecommendAlgorithm.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.cube.cuboid.algorithm;
+
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public abstract class AbstractRecommendAlgorithm implements CuboidRecommendAlgorithm {
+ private static final Logger logger = LoggerFactory.getLogger(AbstractRecommendAlgorithm.class);
+
+ private CuboidStats cuboidStats;
+ private BenefitPolicy benefitPolicy;
+
+ private AtomicBoolean cancelRequested = new AtomicBoolean(false);
+ private AtomicBoolean canceled = new AtomicBoolean(false);
+
+ private long timeoutMillis;
+
+ public AbstractRecommendAlgorithm(final long timeout, BenefitPolicy benefitPolicy, CuboidStats cuboidStats) {
+ if (timeout <= 0) {
+ this.timeoutMillis = Long.MAX_VALUE;
+ } else {
+ this.timeoutMillis = timeout;
+ }
+ this.cuboidStats = cuboidStats;
+ this.benefitPolicy = benefitPolicy;
+ }
+
+ @Override
+ public void cancel() {
+ cancelRequested.set(true);
+ }
+
+ /**
+ * Checks whether the algorithm has been canceled or timed out.
+ *
+ */
+ protected boolean shouldCancel() {
+ if (canceled.get()) {
+ return true;
+ }
+ if (cancelRequested.get()) {
+ canceled.set(true);
+ cancelRequested.set(false);
+ logger.warn("Algorithm is canceled.");
+ return true;
+ }
+ final long currentTimeMillis = System.currentTimeMillis();
+ if (currentTimeMillis > timeoutMillis) {
+ canceled.set(true);
+ logger.warn("Algorithm exceeds time limit.");
+ return true;
+ }
+ return false;
+ }
+
+ public CuboidStats getCuboidStats() {
+ return cuboidStats;
+ }
+
+ public BenefitPolicy getBenefitPolicy() {
+ return benefitPolicy;
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/e2029dd9/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/BPUSCalculator.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/BPUSCalculator.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/BPUSCalculator.java
new file mode 100755
index 0000000..6d0b654
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/BPUSCalculator.java
@@ -0,0 +1,165 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.cube.cuboid.algorithm;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
+
+/**
+ * Calculate the benefit based on Benefit Per Unit Space.
+ */
+public class BPUSCalculator implements BenefitPolicy {
+
+ private static Logger logger = LoggerFactory.getLogger(BPUSCalculator.class);
+
+ protected CuboidStats cuboidStats;
+ protected Map<Long, Long> cuboidAggCostMap;
+
+ public BPUSCalculator(CuboidStats cuboidStats) {
+ this.cuboidStats = cuboidStats;
+ this.cuboidAggCostMap = Maps.newHashMap();
+ }
+
+ @Override
+ public void initBeforeStart() {
+ cuboidAggCostMap.clear();
+ //Initialize stats for mandatory cuboids
+ for (Long cuboid : cuboidStats.getAllCuboidsForMandatory()) {
+ if (getCuboidCost(cuboid) != null) {
+ cuboidAggCostMap.put(cuboid, getCuboidCost(cuboid));
+ }
+ }
+ Set<Long> mandatoryCuboidSetWithStats = cuboidAggCostMap.keySet();
+ //Initialize stats for selection cuboids
+ long baseCuboidCost = getCuboidCost(cuboidStats.getBaseCuboid());
+ for (Long cuboid : cuboidStats.getAllCuboidsForSelection()) {
+ long leastCost = baseCuboidCost;
+ for (Long cuboidTarget : mandatoryCuboidSetWithStats) {
+ if ((cuboid | cuboidTarget) == cuboidTarget) {
+ if (leastCost > cuboidAggCostMap.get(cuboidTarget)) {
+ leastCost = cuboidAggCostMap.get(cuboidTarget);
+ }
+ }
+ }
+ cuboidAggCostMap.put(cuboid, leastCost);
+ }
+ }
+
+ @Override
+ public CuboidBenefitModel.BenefitModel calculateBenefit(long cuboid, Set<Long> selected) {
+ double totalCostSaving = 0;
+ int benefitCount = 0;
+ for (Long descendant : cuboidStats.getAllDescendants(cuboid)) {
+ if (!selected.contains(descendant)) {
+ double costSaving = getCostSaving(descendant, cuboid);
+ if (costSaving > 0) {
+ totalCostSaving += costSaving;
+ benefitCount++;
+ }
+ }
+ }
+
+ double spaceCost = calculateSpaceCost(cuboid);
+ double benefitPerUnitSpace = totalCostSaving / spaceCost;
+ return new CuboidBenefitModel.BenefitModel(benefitPerUnitSpace, benefitCount);
+ }
+
+ @Override
+ public CuboidBenefitModel.BenefitModel calculateBenefitTotal(List<Long> cuboidsToAdd, Set<Long> selected) {
+ Set<Long> selectedInner = Sets.newHashSet(selected);
+ Map<Long, Long> cuboidAggCostMapSnapshot = Maps.newHashMap(cuboidAggCostMap);
+ for (Long cuboid : cuboidsToAdd) {
+ selectedInner.add(cuboid);
+ propagateAggregationCost(cuboid, selectedInner);
+ }
+ double totalCostSaving = 0;
+ int benefitCount = 0;
+ for (Long cuboid : cuboidAggCostMap.keySet()) {
+ if (cuboidAggCostMap.get(cuboid) < cuboidAggCostMapSnapshot.get(cuboid)) {
+ totalCostSaving += cuboidAggCostMapSnapshot.get(cuboid) - cuboidAggCostMap.get(cuboid);
+ benefitCount++;
+ }
+ }
+ cuboidAggCostMap = cuboidAggCostMapSnapshot;
+
+ double benefitPerUnitSpace = totalCostSaving;
+ return new CuboidBenefitModel.BenefitModel(benefitPerUnitSpace, benefitCount);
+ }
+
+ protected double getCostSaving(long descendant, long cuboid) {
+ long cuboidCost = getCuboidCost(cuboid);
+ long descendantAggCost = getCuboidAggregationCost(descendant);
+ return descendantAggCost - cuboidCost;
+ }
+
+ protected Long getCuboidCost(long cuboid) {
+ return cuboidStats.getCuboidCount(cuboid);
+ }
+
+ private long getCuboidAggregationCost(long cuboid) {
+ return cuboidAggCostMap.get(cuboid);
+ }
+
+ @Override
+ public boolean ifEfficient(CuboidBenefitModel best) {
+ if (best.getBenefit() < getMinBenefitRatio()) {
+ logger.info(String.format("The recommended cuboid %s doesn't meet minimum benifit ratio %f", best,
+ getMinBenefitRatio()));
+ return false;
+ }
+ return true;
+ }
+
+ public double getMinBenefitRatio() {
+ return 0.01;
+ }
+
+ @Override
+ public void propagateAggregationCost(long cuboid, Set<Long> selected) {
+ long aggregationCost = getCuboidCost(cuboid);
+ Set<Long> childrenCuboids = cuboidStats.getAllDescendants(cuboid);
+ for (Long child : childrenCuboids) {
+ if (!selected.contains(child) && (aggregationCost < getCuboidAggregationCost(child))) {
+ cuboidAggCostMap.put(child, aggregationCost);
+ }
+ }
+ }
+
+ /**
+ * Return the space cost of building a cuboid.
+ *
+ */
+ public double calculateSpaceCost(long cuboid) {
+ return cuboidStats.getCuboidCount(cuboid);
+ }
+
+ @Override
+ public BenefitPolicy getInstance() {
+ BPUSCalculator bpusCalculator = new BPUSCalculator(this.cuboidStats);
+ bpusCalculator.cuboidAggCostMap.putAll(this.cuboidAggCostMap);
+ return bpusCalculator;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/e2029dd9/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/BenefitPolicy.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/BenefitPolicy.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/BenefitPolicy.java
new file mode 100755
index 0000000..43bd3af
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/BenefitPolicy.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.cube.cuboid.algorithm;
+
+import java.util.List;
+import java.util.Set;
+
+/**
+* Calculate the benefit of building the cuboid, the already selected cuboids will affect the benefits of the given cuboid
+*
+*/
+public interface BenefitPolicy {
+
+ public BenefitPolicy getInstance();
+
+ public void initBeforeStart();
+
+ public CuboidBenefitModel.BenefitModel calculateBenefit(long cuboid, Set<Long> selected);
+
+ public CuboidBenefitModel.BenefitModel calculateBenefitTotal(List<Long> cuboidsToAdd, Set<Long> selected);
+
+ public boolean ifEfficient(CuboidBenefitModel best);
+
+ public void propagateAggregationCost(long cuboid, Set<Long> selected);
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/e2029dd9/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/CuboidBenefitModel.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/CuboidBenefitModel.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/CuboidBenefitModel.java
new file mode 100755
index 0000000..42f1ecf
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/CuboidBenefitModel.java
@@ -0,0 +1,210 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.cube.cuboid.algorithm;
+
+public class CuboidBenefitModel {
+ private CuboidModel cuboidModel;
+ private BenefitModel benefitModel;
+
+ public CuboidBenefitModel(CuboidModel cuboidModel, BenefitModel benefitModel) {
+ this.cuboidModel = cuboidModel;
+ this.benefitModel = benefitModel;
+ }
+
+ public void reset(CuboidModel cuboidModel, BenefitModel benefitModel) {
+ this.cuboidModel = cuboidModel;
+ this.benefitModel = benefitModel;
+ }
+
+ public Long getCuboidId() {
+ return cuboidModel == null ? null : cuboidModel.getCuboidId();
+ }
+
+ public Double getBenefit() {
+ return benefitModel == null ? null : benefitModel.getBenefit();
+ }
+
+ @Override
+ public String toString() {
+ return "CuboidBenefitModel [cuboidModel=" + cuboidModel + ", benefitModel=" + benefitModel + "]";
+ }
+
+ @Override
+ public int hashCode() {
+ final int prime = 31;
+ int result = 1;
+ result = prime * result + ((cuboidModel == null) ? 0 : cuboidModel.hashCode());
+ result = prime * result + ((benefitModel == null) ? 0 : benefitModel.hashCode());
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj)
+ return true;
+ if (obj == null)
+ return false;
+ if (getClass() != obj.getClass())
+ return false;
+ CuboidBenefitModel other = (CuboidBenefitModel) obj;
+ if (cuboidModel == null) {
+ if (other.cuboidModel != null)
+ return false;
+ } else if (!cuboidModel.equals(other.cuboidModel))
+ return false;
+ if (benefitModel == null) {
+ if (other.benefitModel != null)
+ return false;
+ } else if (!benefitModel.equals(other.benefitModel))
+ return false;
+ return true;
+ }
+
+ public static class CuboidModel {
+ private long cuboidId;
+
+ private long recordCount;
+ private double spaceSize;
+
+ private double hitProbability;
+ private long scanCount;
+
+ public CuboidModel(long cuboId, long recordCount, double spaceSize, double hitProbability, long scanCount) {
+ this.cuboidId = cuboId;
+ this.recordCount = recordCount;
+ this.spaceSize = spaceSize;
+ this.hitProbability = hitProbability;
+ this.scanCount = scanCount;
+ }
+
+ public long getCuboidId() {
+ return cuboidId;
+ }
+
+ public long getRecordCount() {
+ return recordCount;
+ }
+
+ public double getSpaceSize() {
+ return spaceSize;
+ }
+
+ public double getHitProbability() {
+ return hitProbability;
+ }
+
+ public long getScanCount() {
+ return scanCount;
+ }
+
+ @Override
+ public String toString() {
+ return "CuboidModel [cuboidId=" + cuboidId + ", recordCount=" + recordCount + ", spaceSize=" + spaceSize
+ + ", hitProbability=" + hitProbability + ", scanCount=" + scanCount + "]";
+ }
+
+ @Override
+ public int hashCode() {
+ final int prime = 31;
+ int result = 1;
+ result = prime * result + (int) (cuboidId ^ (cuboidId >>> 32));
+ result = prime * result + (int) (recordCount ^ (recordCount >>> 32));
+ long temp;
+ temp = Double.doubleToLongBits(spaceSize);
+ result = prime * result + (int) (temp ^ (temp >>> 32));
+ temp = Double.doubleToLongBits(hitProbability);
+ result = prime * result + (int) (temp ^ (temp >>> 32));
+ result = prime * result + (int) (scanCount ^ (scanCount >>> 32));
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj)
+ return true;
+ if (obj == null)
+ return false;
+ if (getClass() != obj.getClass())
+ return false;
+
+ CuboidModel other = (CuboidModel) obj;
+ if (cuboidId != other.cuboidId)
+ return false;
+ if (recordCount != other.recordCount)
+ return false;
+ if (Double.doubleToLongBits(spaceSize) != Double.doubleToLongBits(other.spaceSize))
+ return false;
+ if (hitProbability != other.hitProbability)
+ return false;
+ if (scanCount != other.scanCount)
+ return false;
+ return true;
+ }
+ }
+
+ public static class BenefitModel {
+ private double benefit;
+ private int benefitCount;
+
+ public BenefitModel(double benefit, int benefitCount) {
+ this.benefit = benefit;
+ this.benefitCount = benefitCount;
+ }
+
+ public double getBenefit() {
+ return benefit;
+ }
+
+ public int getBenefitCount() {
+ return benefitCount;
+ }
+
+ @Override
+ public String toString() {
+ return "BenefitModel [benefit=" + benefit + ", benefitCount=" + benefitCount + "]";
+ }
+
+ @Override
+ public int hashCode() {
+ final int prime = 31;
+ int result = 1;
+ long temp;
+ temp = Double.doubleToLongBits(benefit);
+ result = prime * result + (int) (temp ^ (temp >>> 32));
+ result = prime * result + benefitCount;
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj)
+ return true;
+ if (obj == null)
+ return false;
+ if (getClass() != obj.getClass())
+ return false;
+ BenefitModel other = (BenefitModel) obj;
+ if (Double.doubleToLongBits(benefit) != Double.doubleToLongBits(other.benefit))
+ return false;
+ if (benefitCount != other.benefitCount)
+ return false;
+ return true;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/e2029dd9/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/CuboidRecommendAlgorithm.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/CuboidRecommendAlgorithm.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/CuboidRecommendAlgorithm.java
new file mode 100755
index 0000000..b9f39ec
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/CuboidRecommendAlgorithm.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.cube.cuboid.algorithm;
+
+import java.util.List;
+
+/**
+ * Algorithm to calculate the cuboid benifit and recommend cost-effective cuboid list based on the cube statistics.
+ */
+public interface CuboidRecommendAlgorithm {
+
+ /**
+ * Return a list of recommended cuboids for the building segment based on expansionRate
+ *
+ */
+ List<Long> recommend(double expansionRate);
+
+ /**
+ * Start the Algorithm running
+ *
+ * @param maxSpaceLimit
+ * @return
+ */
+ List<Long> start(double maxSpaceLimit);
+
+ /**
+ * Cancel the Algorithm running
+ *
+ * Users can call this method from another thread to can the Algorithm and return the result earlier
+ *
+ */
+ void cancel();
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/e2029dd9/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/CuboidStats.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/CuboidStats.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/CuboidStats.java
new file mode 100755
index 0000000..1775d5a
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/CuboidStats.java
@@ -0,0 +1,272 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.cube.cuboid.algorithm;
+
+import java.util.Map;
+import java.util.Set;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
+
+public class CuboidStats {
+ private static final Logger logger = LoggerFactory.getLogger(CuboidStats.class);
+
+ public static class Builder {
+
+ private static final long THRESHOLD_ROLL_UP_FOR_MANDATORY = 1000L;
+
+ // Required parameters
+ private String key;
+ private Long baseCuboid;
+ private Map<Long, Long> statistics;
+ private Map<Long, Double> size;
+
+ // Optional parameters - initialized to default values
+ private Set<Long> mandatoryCuboids = null;
+ //// These two properties are for generating mandatory cuboids
+ private Map<Long, Map<Long, Long>> rollingUpCountSourceMap = null;
+ private Long rollUpThresholdForMandatory = null;
+
+ private Map<Long, Long> hitFrequencyMap = null;
+ private Map<Long, Map<Long, Long>> scanCountSourceMap = null;
+
+ public Builder(String key, Long baseCuboid, Map<Long, Long> statistics, Map<Long, Double> size) {
+ this.key = key;
+ this.baseCuboid = baseCuboid;
+ this.statistics = statistics;
+ this.size = size;
+ }
+
+ public Builder setRollingUpCountSourceMap(Map<Long, Map<Long, Long>> rollingUpCountSourceMap) {
+ this.rollingUpCountSourceMap = rollingUpCountSourceMap;
+ this.rollUpThresholdForMandatory = THRESHOLD_ROLL_UP_FOR_MANDATORY;
+ return this;
+ }
+
+ public Builder setRollingUpCountSourceMap(Map<Long, Map<Long, Long>> rollingUpCountSourceMap,
+ long rollUpThresholdForMandatory) {
+ this.rollingUpCountSourceMap = rollingUpCountSourceMap;
+ this.rollUpThresholdForMandatory = rollUpThresholdForMandatory;
+ return this;
+ }
+
+ public Builder setMandatoryCuboids(Set<Long> mandatoryCuboids) {
+ this.mandatoryCuboids = mandatoryCuboids;
+ return this;
+ }
+
+ public Builder setHitFrequencyMap(Map<Long, Long> hitFrequencyMap) {
+ this.hitFrequencyMap = hitFrequencyMap;
+ return this;
+ }
+
+ public Builder setScanCountSourceMap(Map<Long, Map<Long, Long>> scanCountSourceMap) {
+ this.scanCountSourceMap = scanCountSourceMap;
+ return this;
+ }
+
+ public CuboidStats build() {
+ Preconditions.checkNotNull(key, "key should not be null");
+ Preconditions.checkNotNull(baseCuboid, "baseCuboid should not be null");
+ Preconditions.checkNotNull(statistics, "statistics should not be null");
+ Preconditions.checkNotNull(size, "size should not be null");
+ Preconditions.checkNotNull(statistics.get(baseCuboid),
+ "row count should exist for base cuboid " + baseCuboid);
+ Preconditions.checkState(statistics.keySet().equals(size.keySet()),
+ "statistics & size should own the same key set");
+ if (mandatoryCuboids == null) {
+ mandatoryCuboids = Sets.newHashSet();
+ }
+ if (rollingUpCountSourceMap != null) {
+ mandatoryCuboids.addAll(CuboidStatsUtil.generateMandatoryCuboidSet(statistics, hitFrequencyMap,
+ rollingUpCountSourceMap, rollUpThresholdForMandatory));
+ }
+
+ return new CuboidStats(key, baseCuboid, mandatoryCuboids, statistics, size, hitFrequencyMap,
+ scanCountSourceMap);
+ }
+ }
+
+ private static final double WEIGHT_FOR_UN_QUERY = 0.2;
+
+ private String key;
+ private long baseCuboid;
+ private ImmutableSet<Long> mandatoryCuboidSet;
+ private ImmutableSet<Long> selectionCuboidSet;
+ private ImmutableMap<Long, Long> cuboidCountMap;
+ private ImmutableMap<Long, Double> cuboidSizeMap;
+ private ImmutableMap<Long, Double> cuboidHitProbabilityMap;
+ private ImmutableMap<Long, Long> cuboidScanCountMap;
+
+ private ImmutableMap<Long, Set<Long>> allDescendantsCache;
+
+ private CuboidStats(String key, long baseCuboidId, Set<Long> mandatoryCuboids, Map<Long, Long> statistics,
+ Map<Long, Double> size, Map<Long, Long> hitFrequencyMap, Map<Long, Map<Long, Long>> scanCountSourceMap) {
+
+ this.key = key;
+ this.baseCuboid = baseCuboidId;
+ /** Initial mandatory cuboids */
+ Set<Long> cuboidsForMandatory = Sets.newHashSet(mandatoryCuboids);
+ //Always add base cuboid.
+ if (!cuboidsForMandatory.contains(baseCuboid)) {
+ cuboidsForMandatory.add(baseCuboid);
+ }
+ logger.info("Mandatory cuboids: " + cuboidsForMandatory);
+
+ /** Initial selection cuboids */
+ Set<Long> cuboidsForSelection = Sets.newHashSet(statistics.keySet());
+ cuboidsForSelection.removeAll(cuboidsForMandatory);
+
+ //There's no overlap between mandatoryCuboidSet and selectionCuboidSet
+ this.mandatoryCuboidSet = ImmutableSet.<Long> builder().addAll(cuboidsForMandatory).build();
+ this.selectionCuboidSet = ImmutableSet.<Long> builder().addAll(cuboidsForSelection).build();
+ if (selectionCuboidSet.isEmpty()) {
+ logger.warn("The selection set should not be empty!!!");
+ }
+
+ /** Initialize row count for mandatory cuboids */
+ CuboidStatsUtil.complementRowCountForMandatoryCuboids(statistics, baseCuboid, mandatoryCuboidSet);
+
+ this.cuboidCountMap = ImmutableMap.<Long, Long> builder().putAll(statistics).build();
+ this.cuboidSizeMap = ImmutableMap.<Long, Double> builder().putAll(size).build();
+
+ /** Initialize the hit probability for each selection cuboid */
+ Map<Long, Double> tmpCuboidHitProbabilityMap = Maps.newHashMapWithExpectedSize(selectionCuboidSet.size());
+ if (hitFrequencyMap != null) {
+ long totalHitFrequency = 0L;
+ for (Map.Entry<Long, Long> hitFrequency : hitFrequencyMap.entrySet()) {
+ if (selectionCuboidSet.contains(hitFrequency.getKey())) {
+ totalHitFrequency += hitFrequency.getValue();
+ }
+ }
+
+ final double unitUncertainProb = WEIGHT_FOR_UN_QUERY / selectionCuboidSet.size();
+ for (Long cuboid : selectionCuboidSet) {
+ //Calculate hit probability for each cuboid
+ if (hitFrequencyMap.get(cuboid) != null) {
+ tmpCuboidHitProbabilityMap.put(cuboid, unitUncertainProb
+ + (1 - WEIGHT_FOR_UN_QUERY) * hitFrequencyMap.get(cuboid) / totalHitFrequency);
+ } else {
+ tmpCuboidHitProbabilityMap.put(cuboid, unitUncertainProb);
+ }
+ }
+ } else {
+ for (Long cuboid : selectionCuboidSet) {
+ tmpCuboidHitProbabilityMap.put(cuboid, 1.0 / selectionCuboidSet.size());
+ }
+ }
+ this.cuboidHitProbabilityMap = ImmutableMap.<Long, Double> builder().putAll(tmpCuboidHitProbabilityMap).build();
+
+ /** Initialize the scan count when query for each selection cuboid + one base cuboid */
+ Map<Long, Long> tmpCuboidScanCountMap = Maps.newHashMapWithExpectedSize(1 + selectionCuboidSet.size());
+ tmpCuboidScanCountMap.put(baseCuboid, getExpScanCount(baseCuboid, statistics, scanCountSourceMap));
+ for (Long cuboid : selectionCuboidSet) {
+ tmpCuboidScanCountMap.put(cuboid, getExpScanCount(cuboid, statistics, scanCountSourceMap));
+ }
+ this.cuboidScanCountMap = ImmutableMap.<Long, Long> builder().putAll(tmpCuboidScanCountMap).build();
+
+ this.allDescendantsCache = ImmutableMap.<Long, Set<Long>> builder()
+ .putAll(CuboidStatsUtil.createAllDescendantsCache(statistics.keySet())).build();
+ }
+
+ private long getExpScanCount(long sourceCuboid, Map<Long, Long> statistics,
+ Map<Long, Map<Long, Long>> scanCountSourceMap) {
+ Preconditions.checkNotNull(statistics.get(sourceCuboid),
+ "The statistics for source cuboid " + sourceCuboid + " does not exist!!!");
+ if (scanCountSourceMap == null || scanCountSourceMap.get(sourceCuboid) == null
+ || scanCountSourceMap.get(sourceCuboid).size() <= 0) {
+ return statistics.get(sourceCuboid);
+ } else {
+ //TODO some improvement can be done by assigning weights based on distance between source cuboid and target cuboid
+ Map<Long, Long> scanCountTargetMap = scanCountSourceMap.get(sourceCuboid);
+ long totalEstScanCount = 0L;
+ for (Map.Entry<Long, Long> subEntry : scanCountTargetMap.entrySet()) {
+ long targetCuboid = subEntry.getKey();
+ Preconditions.checkNotNull(statistics.get(targetCuboid),
+ "The statistics for target cuboid " + targetCuboid + " does not exist!!!");
+ // Consider the ratio of row count between source cuboid and target cuboid
+ totalEstScanCount += subEntry.getValue() * statistics.get(sourceCuboid) / statistics.get(targetCuboid);
+ }
+ return totalEstScanCount / scanCountTargetMap.size();
+ }
+ }
+
+ public Set<Long> getAllDescendants(long cuboid) {
+ Set<Long> allDescendants = Sets.newLinkedHashSet();
+ if (selectionCuboidSet.contains(cuboid)) {
+ return allDescendantsCache.get(cuboid);
+ }
+ return allDescendants;
+ }
+
+ public Set<Long> getAllCuboidsForSelection() {
+ return selectionCuboidSet;
+ }
+
+ public Set<Long> getAllCuboidsForMandatory() {
+ return mandatoryCuboidSet;
+ }
+
+ public Long getCuboidQueryCost(long cuboid) {
+ return cuboidScanCountMap.get(cuboid);
+ }
+
+ public Long getCuboidCount(long cuboid) {
+ return cuboidCountMap.get(cuboid);
+ }
+
+ public Double getCuboidSize(long cuboid) {
+ return cuboidSizeMap.get(cuboid);
+ }
+
+ public double getCuboidHitProbability(long cuboid) {
+ if (mandatoryCuboidSet.contains(cuboid)) {
+ return 1;
+ } else {
+ return cuboidHitProbabilityMap.get(cuboid) == null ? 0 : cuboidHitProbabilityMap.get(cuboid);
+ }
+ }
+
+ public Map<Long, Long> getStatistics() {
+ return cuboidCountMap;
+ }
+
+ public double getBaseCuboidSize() {
+ return getCuboidSize(baseCuboid);
+ }
+
+ public long getBaseCuboid() {
+ return baseCuboid;
+ }
+
+ public String getKey() {
+ return key;
+ }
+
+ public CuboidBenefitModel.CuboidModel getCuboidModel(long cuboid) {
+ return new CuboidBenefitModel.CuboidModel(cuboid, getCuboidCount(cuboid), getCuboidSize(cuboid),
+ getCuboidHitProbability(cuboid), getCuboidQueryCost(cuboid));
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/e2029dd9/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/CuboidStatsUtil.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/CuboidStatsUtil.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/CuboidStatsUtil.java
new file mode 100644
index 0000000..6d5bbe5
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/CuboidStatsUtil.java
@@ -0,0 +1,166 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.cube.cuboid.algorithm;
+
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.SortedSet;
+import java.util.TreeSet;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
+
+public class CuboidStatsUtil {
+
+ /**
+ * For generating mandatory cuboids,
+ * a cuboid is mandatory if the expectation of rolling up count exceeds a threshold
+ * */
+ public static Set<Long> generateMandatoryCuboidSet(Map<Long, Long> statistics, Map<Long, Long> hitFrequencyMap,
+ Map<Long, Map<Long, Long>> rollingUpCountSourceMap, final long rollUpThresholdForMandatory) {
+ Set<Long> mandatoryCuboidSet = Sets.newHashSet();
+ if (hitFrequencyMap == null || hitFrequencyMap.isEmpty() || rollingUpCountSourceMap == null
+ || rollingUpCountSourceMap.isEmpty()) {
+ return mandatoryCuboidSet;
+ }
+ long totalHitFrequency = 0L;
+ for (long hitFrequency : hitFrequencyMap.values()) {
+ totalHitFrequency += hitFrequency;
+ }
+
+ if (totalHitFrequency == 0) {
+ return mandatoryCuboidSet;
+ }
+
+ for (Map.Entry<Long, Long> hitFrequency : hitFrequencyMap.entrySet()) {
+ long cuboid = hitFrequency.getKey();
+ if (statistics.get(cuboid) != null) {
+ continue;
+ }
+ if (rollingUpCountSourceMap.get(cuboid) == null || rollingUpCountSourceMap.get(cuboid).isEmpty()) {
+ continue;
+ }
+ long totalEstScanCount = 0L;
+ for (long estScanCount : rollingUpCountSourceMap.get(cuboid).values()) {
+ totalEstScanCount += estScanCount;
+ }
+ totalEstScanCount /= rollingUpCountSourceMap.get(cuboid).size();
+ if ((hitFrequency.getValue() * 1.0 / totalHitFrequency)
+ * totalEstScanCount >= rollUpThresholdForMandatory) {
+ mandatoryCuboidSet.add(cuboid);
+ }
+ }
+ return mandatoryCuboidSet;
+ }
+
+ /**
+ * Complement row count for mandatory cuboids
+ * with its best parent's row count
+ * */
+ public static void complementRowCountForMandatoryCuboids(Map<Long, Long> statistics, long baseCuboid,
+ Set<Long> mandatoryCuboidSet) {
+ // Sort entries order by row count asc
+ SortedSet<Map.Entry<Long, Long>> sortedStatsSet = new TreeSet<Map.Entry<Long, Long>>(
+ new Comparator<Map.Entry<Long, Long>>() {
+ public int compare(Map.Entry<Long, Long> o1, Map.Entry<Long, Long> o2) {
+ return o1.getValue().compareTo(o2.getValue());
+ }
+ });
+ sortedStatsSet.addAll(statistics.entrySet());
+ for (Long cuboid : mandatoryCuboidSet) {
+ if (statistics.get(cuboid) == null) {
+ // Get estimate row count for mandatory cuboid
+ long tmpRowCount = -1;
+ for (Map.Entry<Long, Long> entry : sortedStatsSet) {
+ if (isDescendant(cuboid, entry.getKey())) {
+ tmpRowCount = entry.getValue();
+ }
+ }
+ statistics.put(cuboid, tmpRowCount < 0 ? statistics.get(baseCuboid) : tmpRowCount);
+ }
+ }
+ }
+
+ /** Using dynamic programming to use extra space to reduce repetitive computation*/
+ public static Map<Long, Set<Long>> createAllDescendantsCache(final Set<Long> cuboidSet) {
+ List<Long> latticeCuboidList = Lists.newArrayList(cuboidSet);
+ Collections.sort(latticeCuboidList);
+
+ Map<Long, Set<Long>> allDescendantsCache = Maps.newHashMap();
+ Set<Long> preNoneDescendants = Sets.newHashSet();
+ for (int i = 0; i < latticeCuboidList.size(); i++) {
+ Long currentCuboid = latticeCuboidList.get(i);
+ Set<Long> descendants = Sets.newHashSet(currentCuboid);
+ Set<Long> curNoneDescendants = Sets.newHashSet();
+ if (i > 0) {
+ long preCuboid = latticeCuboidList.get(i - 1);
+ if (isDescendant(preCuboid, currentCuboid)) {
+ descendants.addAll(allDescendantsCache.get(preCuboid));
+ } else {
+ curNoneDescendants.add(preCuboid);
+ for (long cuboidToCheck : allDescendantsCache.get(preCuboid)) {
+ if (isDescendant(cuboidToCheck, currentCuboid)) {
+ descendants.addAll(allDescendantsCache.get(cuboidToCheck));
+ }
+ }
+ }
+ }
+ for (long cuboidToCheck : preNoneDescendants) {
+ if (isDescendant(cuboidToCheck, currentCuboid)) {
+ descendants.addAll(allDescendantsCache.get(cuboidToCheck));
+ } else {
+ curNoneDescendants.add(cuboidToCheck);
+ }
+ }
+
+ allDescendantsCache.put(currentCuboid, descendants);
+ preNoneDescendants = curNoneDescendants;
+ }
+
+ return allDescendantsCache;
+ }
+
+ @VisibleForTesting
+ static Map<Long, Set<Long>> createAllDescendantsCache2(final Set<Long> cuboidSet) {
+ List<Long> latticeCuboidList = Lists.newArrayList(cuboidSet);
+
+ Map<Long, Set<Long>> allDescendantsCache = Maps.newHashMap();
+ for (int i = 0; i < latticeCuboidList.size(); i++) {
+ Long currentCuboid = latticeCuboidList.get(i);
+ Set<Long> descendantSet = Sets.newHashSet(currentCuboid);
+ for (int j = 0; j < i; j++) {
+ Long checkCuboid = latticeCuboidList.get(j);
+ if (isDescendant(checkCuboid, currentCuboid)) {
+ descendantSet.add(checkCuboid);
+ }
+ }
+ allDescendantsCache.put(currentCuboid, descendantSet);
+ }
+ return allDescendantsCache;
+ }
+
+ public static boolean isDescendant(long cuboidToCheck, long parentCuboid) {
+ return (cuboidToCheck & parentCuboid) == cuboidToCheck;
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/e2029dd9/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/PBPUSCalculator.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/PBPUSCalculator.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/PBPUSCalculator.java
new file mode 100755
index 0000000..6d3ddc7
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/PBPUSCalculator.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.cube.cuboid.algorithm;
+
+import com.google.common.base.Preconditions;
+
+/**
+ * Calculate the benefit based on Benefit Per Unit Space with query probability distribution.
+ */
+public class PBPUSCalculator extends BPUSCalculator {
+
+ public PBPUSCalculator(final CuboidStats cuboidStats) {
+ super(cuboidStats);
+ }
+
+ @Override
+ protected double getCostSaving(long descendant, long cuboid) {
+ return getCuboidHitProbability(descendant) * super.getCostSaving(descendant, cuboid);
+ }
+
+ protected double getCuboidHitProbability(long cuboid) {
+ return cuboidStats.getCuboidHitProbability(cuboid);
+ }
+
+ public double getMinBenefitRatio() {
+ Preconditions.checkArgument(cuboidStats.getAllCuboidsForSelection().size() > 0,
+ "The set of cuboids for selection is empty!!!");
+ return super.getMinBenefitRatio() / cuboidStats.getAllCuboidsForSelection().size();
+ }
+
+ @Override
+ public BenefitPolicy getInstance() {
+ PBPUSCalculator pbpusCalculator = new PBPUSCalculator(cuboidStats);
+ pbpusCalculator.cuboidAggCostMap.putAll(this.cuboidAggCostMap);
+ return pbpusCalculator;
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/e2029dd9/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/SPBPUSCalculator.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/SPBPUSCalculator.java b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/SPBPUSCalculator.java
new file mode 100755
index 0000000..c89bf49
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/cube/cuboid/algorithm/SPBPUSCalculator.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.cube.cuboid.algorithm;
+
+/**
+ * Calculate the benefit based on Benefit Per Unit Space with query probability distribution and updated cost.
+ */
+public class SPBPUSCalculator extends PBPUSCalculator {
+
+ public SPBPUSCalculator(final CuboidStats cuboidStats) {
+ super(cuboidStats);
+ }
+
+ @Override
+ protected Long getCuboidCost(long cuboid) {
+ return cuboidStats.getCuboidQueryCost(cuboid);
+ }
+
+ @Override
+ public BenefitPolicy getInstance() {
+ SPBPUSCalculator spbpusCalculator = new SPBPUSCalculator(cuboidStats);
+ spbpusCalculator.cuboidAggCostMap.putAll(this.cuboidAggCostMap);
+ return spbpusCalculator;
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/e2029dd9/core-cube/src/test/java/org/apache/kylin/cube/cuboid/algorithm/AlgorithmTestBase.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/cuboid/algorithm/AlgorithmTestBase.java b/core-cube/src/test/java/org/apache/kylin/cube/cuboid/algorithm/AlgorithmTestBase.java
new file mode 100755
index 0000000..02927c3
--- /dev/null
+++ b/core-cube/src/test/java/org/apache/kylin/cube/cuboid/algorithm/AlgorithmTestBase.java
@@ -0,0 +1,310 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.cube.cuboid.algorithm;
+
+import java.io.BufferedReader;
+import java.io.FileReader;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.kylin.cube.cuboid.TreeCuboidScheduler.CuboidCostComparator;
+import org.apache.kylin.cube.cuboid.TreeCuboidScheduler.CuboidTree;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
+
+public class AlgorithmTestBase {
+
+ public CuboidStats cuboidStats;
+
+ private Set<Long> mandatoryCuboids;
+
+ @Before
+ public void setUp() throws Exception {
+
+ mandatoryCuboids = Sets.newHashSet();
+ mandatoryCuboids.add(3000L);
+ mandatoryCuboids.add(1888L);
+ mandatoryCuboids.add(88L);
+ cuboidStats = new CuboidStats.Builder("test", 4095L, simulateCount(), simulateSpaceSize())
+ .setMandatoryCuboids(mandatoryCuboids).setHitFrequencyMap(simulateHitFrequency())
+ .setScanCountSourceMap(simulateScanCount()).build();
+ }
+
+ @After
+ public void after() throws Exception {
+ }
+
+ @Test
+ public void testMandatoryRowCountEstimation() {
+ for (long mandatoryCuboid : mandatoryCuboids) {
+ System.out.println("Cuboid id: " + mandatoryCuboid + "; Row Count: "
+ + cuboidStats.getStatistics().get(mandatoryCuboid));
+ }
+ }
+
+ /** better if closer to 1, worse if closer to 0*/
+ public double getQueryCostRatio(CuboidStats cuboidStats, List<Long> recommendList) {
+ CuboidTree cuboidTree = CuboidTree.createFromCuboids(recommendList,
+ new CuboidCostComparator(cuboidStats.getStatistics()));
+ double queryCostBest = 0;
+ for (Long cuboidId : cuboidStats.getAllCuboidsForSelection()) {
+ if (cuboidStats.getCuboidQueryCost(cuboidId) != null) {
+ queryCostBest += cuboidStats.getCuboidHitProbability(cuboidId) * cuboidStats.getCuboidCount(cuboidId);
+ // queryCostBest += cuboidStats.getCuboidHitProbability(cuboidId) * cuboidStats.getCuboidQueryCost(cuboidId);
+ }
+ }
+
+ double queryCost = 0;
+ for (Long cuboidId : cuboidStats.getAllCuboidsForSelection()) {
+ long matchCuboidId = cuboidTree.findBestMatchCuboid(cuboidId);
+ if (cuboidStats.getCuboidQueryCost(matchCuboidId) != null) {
+ queryCost += cuboidStats.getCuboidHitProbability(cuboidId) * cuboidStats.getCuboidCount(matchCuboidId);
+ // queryCost += cuboidStats.getCuboidHitProbability(cuboidId) * cuboidStats.getCuboidQueryCost(matchCuboidId);
+ }
+ }
+
+ return queryCostBest / queryCost;
+ }
+
+ protected Map<Long, Long> simulateCount() {
+ Map<Long, Long> countMap = Maps.newHashMap();
+ BufferedReader br = null;
+
+ try {
+
+ String sCurrentLine;
+
+ br = new BufferedReader(new FileReader("src/test/resources/statistics.txt"));
+
+ while ((sCurrentLine = br.readLine()) != null) {
+ String[] statPair = sCurrentLine.split(" ");
+ countMap.put(Long.valueOf(statPair[0]), Long.valueOf(statPair[1]));
+ }
+
+ } catch (IOException e) {
+ e.printStackTrace();
+ } finally {
+ try {
+ if (br != null)
+ br.close();
+ } catch (IOException ex) {
+ ex.printStackTrace();
+ }
+ }
+
+ return countMap;
+ }
+
+ protected Map<Long, Double> simulateSpaceSize() {
+ Map<Long, Double> sizeMap = Maps.newHashMap();
+ Map<Long, Long> countMap = simulateCount();
+ for (Map.Entry<Long, Long> entry : countMap.entrySet()) {
+ sizeMap.put(entry.getKey(), entry.getValue() * 1.0);
+ }
+ return sizeMap;
+ }
+
+ protected Map<Long, Long> simulateHitFrequency() {
+ Map<Long, Long> hitFrequencyMap = Maps.newHashMap();
+
+ hitFrequencyMap.put(4095L, 10L);
+ hitFrequencyMap.put(3849L, 15L);
+ hitFrequencyMap.put(3780L, 31L);
+
+ hitFrequencyMap.put(3459L, 16L);
+ hitFrequencyMap.put(3145L, 29L);
+
+ hitFrequencyMap.put(2861L, 21L);
+ hitFrequencyMap.put(2768L, 40L);
+
+ hitFrequencyMap.put(1528L, 10L);
+ hitFrequencyMap.put(1440L, 9L);
+ hitFrequencyMap.put(1152L, 21L);
+
+ hitFrequencyMap.put(256L, 23L);
+
+ hitFrequencyMap.put(128L, 7L);
+ hitFrequencyMap.put(272L, 8L);
+ hitFrequencyMap.put(288L, 10L);
+ hitFrequencyMap.put(384L, 2L);
+ hitFrequencyMap.put(320L, 3L);
+ hitFrequencyMap.put(432L, 5L);
+ hitFrequencyMap.put(258L, 8L);
+ hitFrequencyMap.put(336L, 10L);
+ hitFrequencyMap.put(274L, 22L);
+ hitFrequencyMap.put(488L, 41L);
+ hitFrequencyMap.put(352L, 10L);
+
+ hitFrequencyMap.put(16L, 1L);
+ hitFrequencyMap.put(32L, 5L);
+ hitFrequencyMap.put(34L, 1L);
+
+ hitFrequencyMap.put(2L, 21L);
+
+ return hitFrequencyMap;
+ }
+
+ protected Map<Long, Map<Long, Long>> simulateScanCount() {
+ Map<Long, Map<Long, Long>> scanCountMap = Maps.newLinkedHashMap();
+ scanCountMap.put(4094L, new HashMap<Long, Long>() {
+ {
+ put(4095L, 1833041L);
+ }
+ });
+ scanCountMap.put(3849L, new HashMap<Long, Long>() {
+ {
+ put(3849L, 276711L);
+ }
+ });
+ scanCountMap.put(3780L, new HashMap<Long, Long>() {
+ {
+ put(3780L, 129199L);
+ }
+ });
+ scanCountMap.put(3459L, new HashMap<Long, Long>() {
+ {
+ put(3459L, 168109L);
+ }
+ });
+ scanCountMap.put(3145L, new HashMap<Long, Long>() {
+ {
+ put(3145L, 299991L);
+ }
+ });
+ scanCountMap.put(2861L, new HashMap<Long, Long>() {
+ {
+ put(2861L, 2121L);
+ }
+ });
+ scanCountMap.put(2768L, new HashMap<Long, Long>() {
+ {
+ put(2768L, 40231L);
+ }
+ });
+ scanCountMap.put(256L, new HashMap<Long, Long>() {
+ {
+ put(256L, 1L);
+ }
+ });
+ scanCountMap.put(16L, new HashMap<Long, Long>() {
+ {
+ put(16L, 1L);
+ }
+ });
+ scanCountMap.put(32L, new HashMap<Long, Long>() {
+ {
+ put(32L, 2L);
+ }
+ });
+ scanCountMap.put(128L, new HashMap<Long, Long>() {
+ {
+ put(128L, 3L);
+ }
+ });
+ scanCountMap.put(272L, new HashMap<Long, Long>() {
+ {
+ put(272L, 2L);
+ }
+ });
+ scanCountMap.put(288L, new HashMap<Long, Long>() {
+ {
+ put(288L, 3L);
+ }
+ });
+ scanCountMap.put(2L, new HashMap<Long, Long>() {
+ {
+ put(2L, 1L);
+ }
+ });
+ scanCountMap.put(384L, new HashMap<Long, Long>() {
+ {
+ put(384L, 2L);
+ }
+ });
+ scanCountMap.put(320L, new HashMap<Long, Long>() {
+ {
+ put(320L, 3L);
+ }
+ });
+ scanCountMap.put(432L, new HashMap<Long, Long>() {
+ {
+ put(432L, 5L);
+ }
+ });
+ scanCountMap.put(1152L, new HashMap<Long, Long>() {
+ {
+ put(1152L, 21L);
+ }
+ });
+ scanCountMap.put(258L, new HashMap<Long, Long>() {
+ {
+ put(258L, 2L);
+ }
+ });
+ scanCountMap.put(1440L, new HashMap<Long, Long>() {
+ {
+ put(1440L, 9L);
+ }
+ });
+ scanCountMap.put(336L, new HashMap<Long, Long>() {
+ {
+ put(336L, 2L);
+ }
+ });
+ scanCountMap.put(336L, new HashMap<Long, Long>() {
+ {
+ put(336L, 2L);
+ }
+ });
+ scanCountMap.put(274L, new HashMap<Long, Long>() {
+ {
+ put(274L, 1L);
+ }
+ });
+ scanCountMap.put(488L, new HashMap<Long, Long>() {
+ {
+ put(488L, 16L);
+ }
+ });
+ scanCountMap.put(352L, new HashMap<Long, Long>() {
+ {
+ put(352L, 3L);
+ }
+ });
+ scanCountMap.put(1528L, new HashMap<Long, Long>() {
+ {
+ put(1528L, 21L);
+ }
+ });
+ scanCountMap.put(34L, new HashMap<Long, Long>() {
+ {
+ put(34L, 1L);
+ }
+ });
+
+ return scanCountMap;
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/e2029dd9/core-cube/src/test/java/org/apache/kylin/cube/cuboid/algorithm/CuboidStatsUtilTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/cuboid/algorithm/CuboidStatsUtilTest.java b/core-cube/src/test/java/org/apache/kylin/cube/cuboid/algorithm/CuboidStatsUtilTest.java
new file mode 100644
index 0000000..ca70820
--- /dev/null
+++ b/core-cube/src/test/java/org/apache/kylin/cube/cuboid/algorithm/CuboidStatsUtilTest.java
@@ -0,0 +1,161 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.cube.cuboid.algorithm;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import com.google.common.base.Stopwatch;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
+
+public class CuboidStatsUtilTest {
+
+ /**
+ * (11111111)
+ * / | \
+ * 10011111 (11101111) 00110010
+ * \ | /
+ * \ 11000111 /
+ * \ | /
+ * 00000110 /
+ * / | /
+ * 00000100 00000010
+ * */
+ private Set<Long> generateCuboidSet() {
+ return Sets.newHashSet(255L, 159L, 239L, 50L, 199L, 6L, 4L, 2L);
+ }
+
+ private Map<Long, Long> simulateCount() {
+ Map<Long, Long> countMap = Maps.newHashMap();
+
+ countMap.put(255L, 10000L);
+ countMap.put(159L, 10000L);
+ countMap.put(50L, 10000L);
+ countMap.put(199L, 10000L);
+ countMap.put(6L, 10000L);
+ countMap.put(4L, 10000L);
+ countMap.put(2L, 10000L);
+
+ return countMap;
+ }
+
+ private Map<Long, Long> simulateHitFrequency() {
+ Map<Long, Long> hitFrequencyMap = Maps.newHashMap();
+
+ long totalHitFrequency = 10000L;
+
+ hitFrequencyMap.put(239L, (long) (totalHitFrequency * 0.5));
+ hitFrequencyMap.put(50L, (long) (totalHitFrequency * 0.2));
+ hitFrequencyMap.put(2L, (long) (totalHitFrequency * 0.25));
+ hitFrequencyMap.put(178L, (long) (totalHitFrequency * 0.05));
+
+ return hitFrequencyMap;
+ }
+
+ private Map<Long, Map<Long, Long>> simulateRollingUpCount() {
+ Map<Long, Map<Long, Long>> rollingUpCountMap = Maps.newLinkedHashMap();
+
+ rollingUpCountMap.put(239L, new HashMap<Long, Long>() {
+ {
+ put(255L, 4000L);
+ }
+ });
+
+ rollingUpCountMap.put(178L, new HashMap<Long, Long>() {
+ {
+ put(255L, 5000L);
+ }
+ });
+
+ return rollingUpCountMap;
+ }
+
+ @Test
+ public void isDescendantTest() {
+ Assert.assertTrue(CuboidStatsUtil.isDescendant(6L, 239L));
+ Assert.assertTrue(!CuboidStatsUtil.isDescendant(4L, 50L));
+ }
+
+ @Test
+ public void generateMandatoryCuboidSetTest() {
+ Set<Long> mandatoryCuboidSet = CuboidStatsUtil.generateMandatoryCuboidSet(simulateCount(),
+ simulateHitFrequency(), simulateRollingUpCount(), 1000L);
+ Assert.assertTrue(mandatoryCuboidSet.contains(239L));
+ Assert.assertTrue(!mandatoryCuboidSet.contains(178L));
+ }
+
+ @Test
+ public void complementRowCountForMandatoryCuboidsTest() {
+ Map<Long, Long> countMap = simulateCount();
+ Set<Long> mandatoryCuboidSet = CuboidStatsUtil.generateMandatoryCuboidSet(countMap, simulateHitFrequency(),
+ simulateRollingUpCount(), 1000L);
+ for (long mandatoryCuboid : mandatoryCuboidSet) {
+ Assert.assertNull(countMap.get(mandatoryCuboid));
+ }
+ CuboidStatsUtil.complementRowCountForMandatoryCuboids(countMap, 255L, mandatoryCuboidSet);
+ for (long mandatoryCuboid : mandatoryCuboidSet) {
+ Assert.assertNotNull(countMap.get(mandatoryCuboid));
+ }
+ Assert.assertTrue(countMap.get(239L) == 10000L);
+ }
+
+ @Test
+ public void createAllDescendantsCacheTest() {
+ Set<Long> cuboidSet = generateCuboidSet();
+ Map<Long, Set<Long>> allDescendantsCache = CuboidStatsUtil.createAllDescendantsCache(cuboidSet);
+
+ Assert.assertTrue(allDescendantsCache.get(255L).containsAll(cuboidSet));
+
+ Assert.assertTrue(allDescendantsCache.get(239L).size() == 5);
+
+ Assert.assertTrue(allDescendantsCache.get(50L).containsAll(Sets.newHashSet(50L, 2L)));
+ Assert.assertTrue(!allDescendantsCache.get(50L).contains(4L));
+ }
+
+ private Set<Long> generateMassCuboidSet() {
+ Set<Long> cuboidSet = Sets.newHashSet();
+ long maxCuboid = (1L << 16);
+ for (long i = 1; i < maxCuboid; i++) {
+ cuboidSet.add(i);
+ }
+ return cuboidSet;
+ }
+
+ @Test
+ public void createAllDescendantsCacheStressTest() {
+ Stopwatch sw = new Stopwatch();
+ sw.start();
+ Set<Long> cuboidSet = generateMassCuboidSet();
+ System.out.println("Time elapsed for creating sorted cuboid list: " + sw.elapsed(TimeUnit.MILLISECONDS));
+ sw.reset();
+ sw.start();
+ CuboidStatsUtil.createAllDescendantsCache(cuboidSet);
+ System.out.println("Time elapsed for creating descendants cache: " + sw.elapsed(TimeUnit.MILLISECONDS));
+ sw.reset();
+ sw.start();
+ CuboidStatsUtil.createAllDescendantsCache2(cuboidSet);
+ System.out.println("Time elapsed for creating descendants cache2: " + sw.elapsed(TimeUnit.MILLISECONDS));
+ }
+}