You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2016/12/24 07:34:59 UTC
[1/7] kylin git commit: minor, rm deprecated code [Forced Update!]
Repository: kylin
Updated Branches:
refs/heads/KYLIN-2314 8cbd6a6c1 -> 4881e282e (forced update)
minor, rm deprecated code
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4496928e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4496928e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4496928e
Branch: refs/heads/KYLIN-2314
Commit: 4496928e31abd002636d7695f0bf118210dae6eb
Parents: 120df2d
Author: zhongjian <ji...@163.com>
Authored: Fri Dec 23 20:48:27 2016 +0800
Committer: zhongjian <ji...@163.com>
Committed: Fri Dec 23 20:48:27 2016 +0800
----------------------------------------------------------------------
webapp/app/partials/cubes/cubes.html | 1 -
1 file changed, 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/4496928e/webapp/app/partials/cubes/cubes.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/cubes/cubes.html b/webapp/app/partials/cubes/cubes.html
index 4be7ecd..7fc3adb 100644
--- a/webapp/app/partials/cubes/cubes.html
+++ b/webapp/app/partials/cubes/cubes.html
@@ -92,7 +92,6 @@
<a ng-click="dropCube(cube)" tooltip="Drop the cube, related jobs and data permanently.">Drop</a></li>
<li ng-if="(userService.hasRole('ROLE_ADMIN') || hasPermission(cube, permissions.ADMINISTRATION.mask, permissions.MANAGEMENT.mask))">
<a ng-click="cubeEdit(cube);">Edit</a></li>
- <li ng-if="cube.streaming && cube.status=='DISABLED' && (userService.hasRole('ROLE_ADMIN') || hasPermission(cube, permissions.ADMINISTRATION.mask, permissions.MANAGEMENT.mask))"></li>
<li ng-if="cube.status!='DESCBROKEN'"><a ng-click="startJobSubmit(cube);">Build</a></li>
<li ng-if="cube.status!='DESCBROKEN'"><a ng-click="startRefresh(cube)">Refresh</a></li>
<li ng-if="cube.status!='DESCBROKEN'"><a ng-click="startMerge(cube)">Merge</a></li>
[5/7] kylin git commit: KYLIN-2314 Use col identity instead of col
name in dictionary refs
Posted by li...@apache.org.
http://git-wip-us.apache.org/repos/asf/kylin/blob/c6c67197/examples/test_case_data/localmeta/data/DEFAULT.TEST_COUNTRY.csv
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/data/DEFAULT.TEST_COUNTRY.csv b/examples/test_case_data/localmeta/data/DEFAULT.TEST_COUNTRY.csv
new file mode 100644
index 0000000..aced0e3
--- /dev/null
+++ b/examples/test_case_data/localmeta/data/DEFAULT.TEST_COUNTRY.csv
@@ -0,0 +1,244 @@
+AD,42.546245,1.601554,Andorra
+AE,23.424076,53.847818,United Arab Emirates
+AF,33.93911,67.709953,Afghanistan
+AG,17.060816,-61.796428,Antigua and Barbuda
+AI,18.220554,-63.068615,Anguilla
+AL,41.153332,20.168331,Albania
+AM,40.069099,45.038189,Armenia
+AN,12.226079,-69.060087,Netherlands Antilles
+AO,-11.202692,17.873887,Angola
+AQ,-75.250973,-0.071389,Antarctica
+AR,-38.416097,-63.616672,Argentina
+AS,-14.270972,-170.132217,American Samoa
+AT,47.516231,14.550072,Austria
+AU,-25.274398,133.775136,Australia
+AW,12.52111,-69.968338,Aruba
+AZ,40.143105,47.576927,Azerbaijan
+BA,43.915886,17.679076,Bosnia and Herzegovina
+BB,13.193887,-59.543198,Barbados
+BD,23.684994,90.356331,Bangladesh
+BE,50.503887,4.469936,Belgium
+BF,12.238333,-1.561593,Burkina Faso
+BG,42.733883,25.48583,Bulgaria
+BH,25.930414,50.637772,Bahrain
+BI,-3.373056,29.918886,Burundi
+BJ,9.30769,2.315834,Benin
+BM,32.321384,-64.75737,Bermuda
+BN,4.535277,114.727669,Brunei
+BO,-16.290154,-63.588653,Bolivia
+BR,-14.235004,-51.92528,Brazil
+BS,25.03428,-77.39628,Bahamas
+BT,27.514162,90.433601,Bhutan
+BV,-54.423199,3.413194,Bouvet Island
+BW,-22.328474,24.684866,Botswana
+BY,53.709807,27.953389,Belarus
+BZ,17.189877,-88.49765,Belize
+CA,56.130366,-106.346771,Canada
+CC,-12.164165,96.870956,Cocos [Keeling] Islands
+CD,-4.038333,21.758664,Congo [DRC]
+CF,6.611111,20.939444,Central African Republic
+CG,-0.228021,15.827659,Congo [Republic]
+CH,46.818188,8.227512,Switzerland
+CI,7.539989,-5.54708,C?te d'Ivoire
+CK,-21.236736,-159.777671,Cook Islands
+CL,-35.675147,-71.542969,Chile
+CM,7.369722,12.354722,Cameroon
+CN,35.86166,104.195397,China
+CO,4.570868,-74.297333,Colombia
+CR,9.748917,-83.753428,Costa Rica
+CU,21.521757,-77.781167,Cuba
+CV,16.002082,-24.013197,Cape Verde
+CX,-10.447525,105.690449,Christmas Island
+CY,35.126413,33.429859,Cyprus
+CZ,49.817492,15.472962,Czech Republic
+DE,51.165691,10.451526,Germany
+DJ,11.825138,42.590275,Djibouti
+DK,56.26392,9.501785,Denmark
+DM,15.414999,-61.370976,Dominica
+DO,18.735693,-70.162651,Dominican Republic
+DZ,28.033886,1.659626,Algeria
+EC,-1.831239,-78.183406,Ecuador
+EE,58.595272,25.013607,Estonia
+EG,26.820553,30.802498,Egypt
+EH,24.215527,-12.885834,Western Sahara
+ER,15.179384,39.782334,Eritrea
+ES,40.463667,-3.74922,Spain
+ET,9.145,40.489673,Ethiopia
+FI,61.92411,25.748151,Finland
+FJ,-16.578193,179.414413,Fiji
+FK,-51.796253,-59.523613,Falkland Islands [Islas Malvinas]
+FM,7.425554,150.550812,Micronesia
+FO,61.892635,-6.911806,Faroe Islands
+FR,46.227638,2.213749,France
+GA,-0.803689,11.609444,Gabon
+GB,55.378051,-3.435973,United Kingdom
+GD,12.262776,-61.604171,Grenada
+GE,42.315407,43.356892,Georgia
+GF,3.933889,-53.125782,French Guiana
+GG,49.465691,-2.585278,Guernsey
+GH,7.946527,-1.023194,Ghana
+GI,36.137741,-5.345374,Gibraltar
+GL,71.706936,-42.604303,Greenland
+GM,13.443182,-15.310139,Gambia
+GN,9.945587,-9.696645,Guinea
+GP,16.995971,-62.067641,Guadeloupe
+GQ,1.650801,10.267895,Equatorial Guinea
+GR,39.074208,21.824312,Greece
+GS,-54.429579,-36.587909,South Georgia and the South Sandwich Islands
+GT,15.783471,-90.230759,Guatemala
+GU,13.444304,144.793731,Guam
+GW,11.803749,-15.180413,Guinea-Bissau
+GY,4.860416,-58.93018,Guyana
+GZ,31.354676,34.308825,Gaza Strip
+HK,22.396428,114.109497,Hong Kong
+HM,-53.08181,73.504158,Heard Island and McDonald Islands
+HN,15.199999,-86.241905,Honduras
+HR,45.1,15.2,Croatia
+HT,18.971187,-72.285215,Haiti
+HU,47.162494,19.503304,Hungary
+ID,-0.789275,113.921327,Indonesia
+IE,53.41291,-8.24389,Ireland
+IL,31.046051,34.851612,Israel
+IM,54.236107,-4.548056,Isle of Man
+IN,20.593684,78.96288,India
+IO,-6.343194,71.876519,British Indian Ocean Territory
+IQ,33.223191,43.679291,Iraq
+IR,32.427908,53.688046,Iran
+IS,64.963051,-19.020835,Iceland
+IT,41.87194,12.56738,Italy
+JE,49.214439,-2.13125,Jersey
+JM,18.109581,-77.297508,Jamaica
+JO,30.585164,36.238414,Jordan
+JP,36.204824,138.252924,Japan
+KE,-0.023559,37.906193,Kenya
+KG,41.20438,74.766098,Kyrgyzstan
+KH,12.565679,104.990963,Cambodia
+KI,-3.370417,-168.734039,Kiribati
+KM,-11.875001,43.872219,Comoros
+KN,17.357822,-62.782998,Saint Kitts and Nevis
+KP,40.339852,127.510093,North Korea
+KR,35.907757,127.766922,South Korea
+KW,29.31166,47.481766,Kuwait
+KY,19.513469,-80.566956,Cayman Islands
+KZ,48.019573,66.923684,Kazakhstan
+LA,19.85627,102.495496,Laos
+LB,33.854721,35.862285,Lebanon
+LC,13.909444,-60.978893,Saint Lucia
+LI,47.166,9.555373,Liechtenstein
+LK,7.873054,80.771797,Sri Lanka
+LR,6.428055,-9.429499,Liberia
+LS,-29.609988,28.233608,Lesotho
+LT,55.169438,23.881275,Lithuania
+LU,49.815273,6.129583,Luxembourg
+LV,56.879635,24.603189,Latvia
+LY,26.3351,17.228331,Libya
+MA,31.791702,-7.09262,Morocco
+MC,43.750298,7.412841,Monaco
+MD,47.411631,28.369885,Moldova
+ME,42.708678,19.37439,Montenegro
+MG,-18.766947,46.869107,Madagascar
+MH,7.131474,171.184478,Marshall Islands
+MK,41.608635,21.745275,Macedonia [FYROM]
+ML,17.570692,-3.996166,Mali
+MM,21.913965,95.956223,Myanmar [Burma]
+MN,46.862496,103.846656,Mongolia
+MO,22.198745,113.543873,Macau
+MP,17.33083,145.38469,Northern Mariana Islands
+MQ,14.641528,-61.024174,Martinique
+MR,21.00789,-10.940835,Mauritania
+MS,16.742498,-62.187366,Montserrat
+MT,35.937496,14.375416,Malta
+MU,-20.348404,57.552152,Mauritius
+MV,3.202778,73.22068,Maldives
+MW,-13.254308,34.301525,Malawi
+MX,23.634501,-102.552784,Mexico
+MY,4.210484,101.975766,Malaysia
+MZ,-18.665695,35.529562,Mozambique
+NA,-22.95764,18.49041,Namibia
+NC,-20.904305,165.618042,New Caledonia
+NE,17.607789,8.081666,Niger
+NF,-29.040835,167.954712,Norfolk Island
+NG,9.081999,8.675277,Nigeria
+NI,12.865416,-85.207229,Nicaragua
+NL,52.132633,5.291266,Netherlands
+NO,60.472024,8.468946,Norway
+NP,28.394857,84.124008,Nepal
+NR,-0.522778,166.931503,Nauru
+NU,-19.054445,-169.867233,Niue
+NZ,-40.900557,174.885971,New Zealand
+OM,21.512583,55.923255,Oman
+PA,8.537981,-80.782127,Panama
+PE,-9.189967,-75.015152,Peru
+PF,-17.679742,-149.406843,French Polynesia
+PG,-6.314993,143.95555,Papua New Guinea
+PH,12.879721,121.774017,Philippines
+PK,30.375321,69.345116,Pakistan
+PL,51.919438,19.145136,Poland
+PM,46.941936,-56.27111,Saint Pierre and Miquelon
+PN,-24.703615,-127.439308,Pitcairn Islands
+PR,18.220833,-66.590149,Puerto Rico
+PS,31.952162,35.233154,Palestinian Territories
+PT,39.399872,-8.224454,Portugal
+PW,7.51498,134.58252,Palau
+PY,-23.442503,-58.443832,Paraguay
+QA,25.354826,51.183884,Qatar
+RE,-21.115141,55.536384,R\ufffd\ufffdunion
+RO,45.943161,24.96676,Romania
+RS,44.016521,21.005859,Serbia
+RU,61.52401,105.318756,Russia
+RW,-1.940278,29.873888,Rwanda
+SA,23.885942,45.079162,Saudi Arabia
+SB,-9.64571,160.156194,Solomon Islands
+SC,-4.679574,55.491977,Seychelles
+SD,12.862807,30.217636,Sudan
+SE,60.128161,18.643501,Sweden
+SG,1.352083,103.819836,Singapore
+SH,-24.143474,-10.030696,Saint Helena
+SI,46.151241,14.995463,Slovenia
+SJ,77.553604,23.670272,Svalbard and Jan Mayen
+SK,48.669026,19.699024,Slovakia
+SL,8.460555,-11.779889,Sierra Leone
+SM,43.94236,12.457777,San Marino
+SN,14.497401,-14.452362,Senegal
+SO,5.152149,46.199616,Somalia
+SR,3.919305,-56.027783,Suriname
+ST,0.18636,6.613081,S?o Tom\ufffd\ufffd and Pr\ufffd\ufffdncipe
+SV,13.794185,-88.89653,El Salvador
+SY,34.802075,38.996815,Syria
+SZ,-26.522503,31.465866,Swaziland
+TC,21.694025,-71.797928,Turks and Caicos Islands
+TD,15.454166,18.732207,Chad
+TF,-49.280366,69.348557,French Southern Territories
+TG,8.619543,0.824782,Togo
+TH,15.870032,100.992541,Thailand
+TJ,38.861034,71.276093,Tajikistan
+TK,-8.967363,-171.855881,Tokelau
+TL,-8.874217,125.727539,Timor-Leste
+TM,38.969719,59.556278,Turkmenistan
+TN,33.886917,9.537499,Tunisia
+TO,-21.178986,-175.198242,Tonga
+TR,38.963745,35.243322,Turkey
+TT,10.691803,-61.222503,Trinidad and Tobago
+TV,-7.109535,177.64933,Tuvalu
+TW,23.69781,120.960515,Taiwan
+TZ,-6.369028,34.888822,Tanzania
+UA,48.379433,31.16558,Ukraine
+UG,1.373333,32.290275,Uganda
+US,37.09024,-95.712891,United States
+UY,-32.522779,-55.765835,Uruguay
+UZ,41.377491,64.585262,Uzbekistan
+VA,41.902916,12.453389,Vatican City
+VC,12.984305,-61.287228,Saint Vincent and the Grenadines
+VE,6.42375,-66.58973,Venezuela
+VG,18.420695,-64.639968,British Virgin Islands
+VI,18.335765,-64.896335,U.S. Virgin Islands
+VN,14.058324,108.277199,Vietnam
+VU,-15.376706,166.959158,Vanuatu
+WF,-13.768752,-177.156097,Wallis and Futuna
+WS,-13.759029,-172.104629,Samoa
+XK,42.602636,20.902977,Kosovo
+YE,15.552727,48.516388,Yemen
+YT,-12.8275,45.166244,Mayotte
+ZA,-30.559482,22.937506,South Africa
+ZM,-13.133897,27.849332,Zambia
+ZW,-19.015438,29.154857,Zimbabwe
http://git-wip-us.apache.org/repos/asf/kylin/blob/c6c67197/examples/test_case_data/localmeta/model_desc/ci_inner_join_model.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/model_desc/ci_inner_join_model.json b/examples/test_case_data/localmeta/model_desc/ci_inner_join_model.json
index d37bb5f..19cf721 100644
--- a/examples/test_case_data/localmeta/model_desc/ci_inner_join_model.json
+++ b/examples/test_case_data/localmeta/model_desc/ci_inner_join_model.json
@@ -17,6 +17,34 @@
}
},
{
+ "table": "DEFAULT.TEST_ACCOUNT",
+ "alias": "BUYER_ACCOUNT",
+ "kind": "FACT",
+ "join": {
+ "type": "INNER",
+ "primary_key": [
+ "BUYER_ACCOUNT.ACCOUNT_ID"
+ ],
+ "foreign_key": [
+ "TEST_ORDER.BUYER_ID"
+ ]
+ }
+ },
+ {
+ "table": "DEFAULT.TEST_ACCOUNT",
+ "alias": "SELLER_ACCOUNT",
+ "kind": "FACT",
+ "join": {
+ "type": "INNER",
+ "primary_key": [
+ "SELLER_ACCOUNT.ACCOUNT_ID"
+ ],
+ "foreign_key": [
+ "TEST_KYLIN_FACT.SELLER_ID"
+ ]
+ }
+ },
+ {
"table": "EDW.TEST_CAL_DT",
"join": {
"type": "INNER",
@@ -65,11 +93,37 @@
"TEST_KYLIN_FACT.SLR_SEGMENT_CD"
]
}
+ },
+ {
+ "table": "DEFAULT.TEST_COUNTRY",
+ "alias": "BUYER_COUNTRY",
+ "join": {
+ "type": "INNER",
+ "primary_key": [
+ "BUYER_COUNTRY.COUNTRY"
+ ],
+ "foreign_key": [
+ "BUYER_ACCOUNT.ACCOUNT_COUNTRY"
+ ]
+ }
+ },
+ {
+ "table": "DEFAULT.TEST_COUNTRY",
+ "alias": "SELLER_COUNTRY",
+ "join": {
+ "type": "INNER",
+ "primary_key": [
+ "SELLER_COUNTRY.COUNTRY"
+ ],
+ "foreign_key": [
+ "SELLER_ACCOUNT.ACCOUNT_COUNTRY"
+ ]
+ }
}
],
"dimensions": [
{
- "table": "DEFAULT.TEST_KYLIN_FACT",
+ "table": "TEST_KYLIN_FACT",
"columns": [
"TRANS_ID",
"ORDER_ID",
@@ -83,7 +137,7 @@
]
},
{
- "table": "DEFAULT.TEST_ORDER",
+ "table": "TEST_ORDER",
"columns": [
"ORDER_ID",
"BUYER_ID",
@@ -93,7 +147,27 @@
]
},
{
- "table": "DEFAULT.TEST_CATEGORY_GROUPINGS",
+ "table": "BUYER_ACCOUNT",
+ "columns": [
+ "ACCOUNT_ID",
+ "ACCOUNT_BUYER_LEVEL",
+ "ACCOUNT_SELLER_LEVEL",
+ "ACCOUNT_COUNTRY",
+ "ACCOUNT_CONTACT"
+ ]
+ },
+ {
+ "table": "SELLER_ACCOUNT",
+ "columns": [
+ "ACCOUNT_ID",
+ "ACCOUNT_BUYER_LEVEL",
+ "ACCOUNT_SELLER_LEVEL",
+ "ACCOUNT_COUNTRY",
+ "ACCOUNT_CONTACT"
+ ]
+ },
+ {
+ "table": "TEST_CATEGORY_GROUPINGS",
"columns": [
"LEAF_CATEG_ID",
"SITE_ID",
@@ -107,7 +181,7 @@
]
},
{
- "table": "EDW.TEST_SITES",
+ "table": "TEST_SITES",
"columns": [
"SITE_ID",
"SITE_NAME",
@@ -115,18 +189,32 @@
]
},
{
- "table": "EDW.TEST_SELLER_TYPE_DIM",
+ "table": "TEST_SELLER_TYPE_DIM",
"columns": [
"SELLER_TYPE_CD",
"SELLER_TYPE_DESC"
]
},
{
- "table": "EDW.TEST_CAL_DT",
+ "table": "TEST_CAL_DT",
"columns": [
"CAL_DT",
"WEEK_BEG_DT"
]
+ },
+ {
+ "table": "BUYER_COUNTRY",
+ "columns": [
+ "COUNTRY",
+ "NAME"
+ ]
+ },
+ {
+ "table": "SELLER_COUNTRY",
+ "columns": [
+ "COUNTRY",
+ "NAME"
+ ]
}
],
"metrics": [
http://git-wip-us.apache.org/repos/asf/kylin/blob/c6c67197/examples/test_case_data/localmeta/model_desc/ci_left_join_model.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/model_desc/ci_left_join_model.json b/examples/test_case_data/localmeta/model_desc/ci_left_join_model.json
index e0f88a7..1b08aaf 100644
--- a/examples/test_case_data/localmeta/model_desc/ci_left_join_model.json
+++ b/examples/test_case_data/localmeta/model_desc/ci_left_join_model.json
@@ -17,6 +17,34 @@
}
},
{
+ "table": "DEFAULT.TEST_ACCOUNT",
+ "alias": "BUYER_ACCOUNT",
+ "kind": "FACT",
+ "join": {
+ "type": "LEFT",
+ "primary_key": [
+ "BUYER_ACCOUNT.ACCOUNT_ID"
+ ],
+ "foreign_key": [
+ "TEST_ORDER.BUYER_ID"
+ ]
+ }
+ },
+ {
+ "table": "DEFAULT.TEST_ACCOUNT",
+ "alias": "SELLER_ACCOUNT",
+ "kind": "FACT",
+ "join": {
+ "type": "LEFT",
+ "primary_key": [
+ "SELLER_ACCOUNT.ACCOUNT_ID"
+ ],
+ "foreign_key": [
+ "TEST_KYLIN_FACT.SELLER_ID"
+ ]
+ }
+ },
+ {
"table": "EDW.TEST_CAL_DT",
"join": {
"type": "LEFT",
@@ -65,11 +93,37 @@
"TEST_KYLIN_FACT.SLR_SEGMENT_CD"
]
}
+ },
+ {
+ "table": "DEFAULT.TEST_COUNTRY",
+ "alias": "BUYER_COUNTRY",
+ "join": {
+ "type": "LEFT",
+ "primary_key": [
+ "BUYER_COUNTRY.COUNTRY"
+ ],
+ "foreign_key": [
+ "BUYER_ACCOUNT.ACCOUNT_COUNTRY"
+ ]
+ }
+ },
+ {
+ "table": "DEFAULT.TEST_COUNTRY",
+ "alias": "SELLER_COUNTRY",
+ "join": {
+ "type": "LEFT",
+ "primary_key": [
+ "SELLER_COUNTRY.COUNTRY"
+ ],
+ "foreign_key": [
+ "SELLER_ACCOUNT.ACCOUNT_COUNTRY"
+ ]
+ }
}
],
"dimensions": [
{
- "table": "DEFAULT.TEST_KYLIN_FACT",
+ "table": "TEST_KYLIN_FACT",
"columns": [
"TRANS_ID",
"ORDER_ID",
@@ -83,7 +137,7 @@
]
},
{
- "table": "DEFAULT.TEST_ORDER",
+ "table": "TEST_ORDER",
"columns": [
"ORDER_ID",
"BUYER_ID",
@@ -93,7 +147,27 @@
]
},
{
- "table": "DEFAULT.TEST_CATEGORY_GROUPINGS",
+ "table": "BUYER_ACCOUNT",
+ "columns": [
+ "ACCOUNT_ID",
+ "ACCOUNT_BUYER_LEVEL",
+ "ACCOUNT_SELLER_LEVEL",
+ "ACCOUNT_COUNTRY",
+ "ACCOUNT_CONTACT"
+ ]
+ },
+ {
+ "table": "SELLER_ACCOUNT",
+ "columns": [
+ "ACCOUNT_ID",
+ "ACCOUNT_BUYER_LEVEL",
+ "ACCOUNT_SELLER_LEVEL",
+ "ACCOUNT_COUNTRY",
+ "ACCOUNT_CONTACT"
+ ]
+ },
+ {
+ "table": "TEST_CATEGORY_GROUPINGS",
"columns": [
"LEAF_CATEG_ID",
"SITE_ID",
@@ -107,7 +181,7 @@
]
},
{
- "table": "EDW.TEST_SITES",
+ "table": "TEST_SITES",
"columns": [
"SITE_ID",
"SITE_NAME",
@@ -115,18 +189,32 @@
]
},
{
- "table": "EDW.TEST_SELLER_TYPE_DIM",
+ "table": "TEST_SELLER_TYPE_DIM",
"columns": [
"SELLER_TYPE_CD",
"SELLER_TYPE_DESC"
]
},
{
- "table": "EDW.TEST_CAL_DT",
+ "table": "TEST_CAL_DT",
"columns": [
"CAL_DT",
"WEEK_BEG_DT"
]
+ },
+ {
+ "table": "BUYER_COUNTRY",
+ "columns": [
+ "COUNTRY",
+ "NAME"
+ ]
+ },
+ {
+ "table": "SELLER_COUNTRY",
+ "columns": [
+ "COUNTRY",
+ "NAME"
+ ]
}
],
"metrics": [
http://git-wip-us.apache.org/repos/asf/kylin/blob/c6c67197/examples/test_case_data/localmeta/model_desc/test_kylin_snowflake_sales_model.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/model_desc/test_kylin_snowflake_sales_model.json b/examples/test_case_data/localmeta/model_desc/test_kylin_snowflake_sales_model.json
deleted file mode 100644
index 84c6979..0000000
--- a/examples/test_case_data/localmeta/model_desc/test_kylin_snowflake_sales_model.json
+++ /dev/null
@@ -1,87 +0,0 @@
-{
- "uuid" : "0928468a-9fab-4185-9a14-6f2e7c74823f",
- "name" : "test_kylin_snowflake_sales_model",
- "lookups" : [ {
- "table" : "SNOWTEST.KYLIN_CAL_DT",
- "join" : {
- "type" : "inner",
- "primary_key" : [ "CAL_DT" ],
- "foreign_key" : [ "PART_DT" ]
- }
- }, {
- "table" : "SNOWTEST.KYLIN_CATEGORY_GROUPINGS",
- "join" : {
- "type" : "inner",
- "primary_key" : [ "LEAF_CATEG_ID", "SITE_ID" ],
- "foreign_key" : [ "LEAF_CATEG_ID", "LSTG_SITE_ID" ]
- }
- }, {
- "table" : "SNOWTEST.KYLIN_ACCOUNT",
- "alias" : "BUYER_ACCOUNT",
- "join" : {
- "type" : "inner",
- "primary_key" : [ "ACCOUNT_ID" ],
- "foreign_key" : [ "BUYER_ID" ]
- }
- }, {
- "table" : "SNOWTEST.KYLIN_ACCOUNT",
- "alias" : "SELLER_ACCOUNT",
- "join" : {
- "type" : "inner",
- "primary_key" : [ "ACCOUNT_ID" ],
- "foreign_key" : [ "SELLER_ID" ]
- }
- }, {
- "table" : "SNOWTEST.KYLIN_COUNTRY",
- "alias" : "BUYER_COUNTRY",
- "join" : {
- "type" : "inner",
- "primary_key" : [ "COUNTRY" ],
- "foreign_key" : [ "BUYER_ACCOUNT.ACCOUNT_COUNTRY" ]
- }
- }, {
- "table" : "SNOWTEST.KYLIN_COUNTRY",
- "alias" : "SELLER_COUNTRY",
- "join" : {
- "type" : "inner",
- "primary_key" : [ "COUNTRY" ],
- "foreign_key" : [ "SELLER_ACCOUNT.ACCOUNT_COUNTRY" ]
- }
- }],
- "dimensions" : [ {
- "table" : "SNOWTEST.KYLIN_SALES",
- "columns" : [ "TRANS_ID", "SELLER_ID", "BUYER_ID", "PART_DT", "LEAF_CATEG_ID", "LSTG_FORMAT_NAME", "LSTG_SITE_ID", "OPS_USER_ID", "OPS_REGION" ]
- }, {
- "table" : "SNOWTEST.KYLIN_CAL_DT",
- "columns" : ["CAL_DT", "WEEK_BEG_DT"]
- }, {
- "table" : "SNOWTEST.KYLIN_CATEGORY_GROUPINGS",
- "columns" : [ "USER_DEFINED_FIELD1", "USER_DEFINED_FIELD3", "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME", "LEAF_CATEG_ID", "SITE_ID" ]
- }, {
- "table" : "BUYER_ACCOUNT",
- "columns" : [ "ACCOUNT_ID", "ACCOUNT_BUYER_LEVEL", "ACCOUNT_SELLER_LEVEL", "ACCOUNT_COUNTRY", "ACCOUNT_CONTACT" ]
- }, {
- "table" : "SELLER_ACCOUNT",
- "columns" : [ "ACCOUNT_ID", "ACCOUNT_BUYER_LEVEL", "ACCOUNT_SELLER_LEVEL", "ACCOUNT_COUNTRY", "ACCOUNT_CONTACT" ]
- }, {
- "table" : "BUYER_COUNTRY",
- "columns" : [ "COUNTRY", "NAME" ]
- }, {
- "table" : "SELLER_COUNTRY",
- "columns" : [ "COUNTRY", "NAME" ]
- } ],
- "metrics": [
- "PRICE",
- "ITEM_COUNT",
- "SELLER_ID",
- "LSTG_FORMAT_NAME"
- ],
- "last_modified" : 1422435345362,
- "fact_table" : "SNOWTEST.KYLIN_SALES",
- "filter_condition" : null,
- "partition_desc" : {
- "partition_date_column" : "SNOWTEST.KYLIN_SALES.PART_DT",
- "partition_date_start" : 1325376000000,
- "partition_type" : "APPEND"
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/c6c67197/examples/test_case_data/localmeta/table/DEFAULT.TEST_ACCOUNT.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/table/DEFAULT.TEST_ACCOUNT.json b/examples/test_case_data/localmeta/table/DEFAULT.TEST_ACCOUNT.json
new file mode 100644
index 0000000..89e265b
--- /dev/null
+++ b/examples/test_case_data/localmeta/table/DEFAULT.TEST_ACCOUNT.json
@@ -0,0 +1,34 @@
+{
+ "uuid" : "f386e39e-40d7-44c2-9eb3-41b365632231",
+
+ "name" : "TEST_ACCOUNT",
+ "data_gen" : "2000",
+
+ "columns" : [ {
+ "id" : "1",
+ "name" : "ACCOUNT_ID",
+ "datatype" : "bigint",
+ "data_gen" : "ID|10000000"
+ }, {
+ "id" : "2",
+ "name" : "ACCOUNT_BUYER_LEVEL",
+ "datatype" : "int",
+ "data_gen" : "RAND||0|5"
+ }, {
+ "id" : "3",
+ "name" : "ACCOUNT_SELLER_LEVEL",
+ "datatype" : "int",
+ "data_gen" : "RAND||0|5"
+ }, {
+ "id" : "4",
+ "name" : "ACCOUNT_COUNTRY",
+ "datatype" : "string",
+ "data_gen" : "CN|FR|GB|GE|JP|IT|RU|US"
+ }, {
+ "id" : "5",
+ "name" : "ACCOUNT_CONTACT",
+ "datatype" : "string"
+ } ],
+ "database" : "DEFAULT",
+ "last_modified" : 0
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/c6c67197/examples/test_case_data/localmeta/table/DEFAULT.TEST_COUNTRY.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/table/DEFAULT.TEST_COUNTRY.json b/examples/test_case_data/localmeta/table/DEFAULT.TEST_COUNTRY.json
new file mode 100644
index 0000000..202182b
--- /dev/null
+++ b/examples/test_case_data/localmeta/table/DEFAULT.TEST_COUNTRY.json
@@ -0,0 +1,24 @@
+{
+ "uuid" : "e286e39e-40d7-44c2-8fa2-41b365632882",
+
+ "name" : "TEST_COUNTRY",
+ "columns" : [ {
+ "id" : "1",
+ "name" : "COUNTRY",
+ "datatype" : "string"
+ }, {
+ "id" : "2",
+ "name" : "LATITUDE",
+ "datatype" : "double"
+ }, {
+ "id" : "3",
+ "name" : "LONGITUDE",
+ "datatype" : "double"
+ }, {
+ "id" : "4",
+ "name" : "NAME",
+ "datatype" : "string"
+ } ],
+ "database" : "DEFAULT",
+ "last_modified" : 0
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/c6c67197/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_COUNTRY.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_COUNTRY.json b/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_COUNTRY.json
deleted file mode 100644
index acf5470..0000000
--- a/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_COUNTRY.json
+++ /dev/null
@@ -1,24 +0,0 @@
-{
- "uuid" : "e286e39e-40d7-44c2-8fa2-41b365632882",
-
- "name" : "TEST_KYLIN_COUNTRY",
- "columns" : [ {
- "id" : "1",
- "name" : "COUNTRY",
- "datatype" : "string"
- }, {
- "id" : "2",
- "name" : "LATITUDE",
- "datatype" : "double"
- }, {
- "id" : "3",
- "name" : "LONGITUDE",
- "datatype" : "double"
- }, {
- "id" : "4",
- "name" : "NAME",
- "datatype" : "string"
- } ],
- "database" : "DEFAULT",
- "last_modified" : 0
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/c6c67197/examples/test_case_data/localmeta/table/SNOWTEST.KYLIN_ACCOUNT.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/table/SNOWTEST.KYLIN_ACCOUNT.json b/examples/test_case_data/localmeta/table/SNOWTEST.KYLIN_ACCOUNT.json
deleted file mode 100644
index 8d39cbc..0000000
--- a/examples/test_case_data/localmeta/table/SNOWTEST.KYLIN_ACCOUNT.json
+++ /dev/null
@@ -1,28 +0,0 @@
-{
- "uuid" : "f386e39e-40d7-44c2-9eb3-41b365632231",
-
- "name" : "KYLIN_ACCOUNT",
- "columns" : [ {
- "id" : "1",
- "name" : "ACCOUNT_ID",
- "datatype" : "bigint"
- }, {
- "id" : "2",
- "name" : "ACCOUNT_BUYER_LEVEL",
- "datatype" : "int"
- }, {
- "id" : "3",
- "name" : "ACCOUNT_SELLER_LEVEL",
- "datatype" : "int"
- }, {
- "id" : "4",
- "name" : "ACCOUNT_COUNTRY",
- "datatype" : "string"
- }, {
- "id" : "5",
- "name" : "ACCOUNT_CONTACT",
- "datatype" : "string"
- } ],
- "database" : "SNOWTEST",
- "last_modified" : 0
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/c6c67197/examples/test_case_data/localmeta/table/SNOWTEST.KYLIN_CAL_DT.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/table/SNOWTEST.KYLIN_CAL_DT.json b/examples/test_case_data/localmeta/table/SNOWTEST.KYLIN_CAL_DT.json
deleted file mode 100644
index d186f76..0000000
--- a/examples/test_case_data/localmeta/table/SNOWTEST.KYLIN_CAL_DT.json
+++ /dev/null
@@ -1,408 +0,0 @@
-{
- "uuid" : "0ff420eb-79ad-40bd-bca9-12d8cd05c60a",
-
- "name" : "KYLIN_CAL_DT",
- "columns" : [ {
- "id" : "1",
- "name" : "CAL_DT",
- "datatype" : "date"
- }, {
- "id" : "2",
- "name" : "YEAR_BEG_DT",
- "datatype" : "date"
- }, {
- "id" : "3",
- "name" : "QTR_BEG_DT",
- "datatype" : "date"
- }, {
- "id" : "4",
- "name" : "MONTH_BEG_DT",
- "datatype" : "date"
- }, {
- "id" : "5",
- "name" : "WEEK_BEG_DT",
- "datatype" : "date"
- }, {
- "id" : "6",
- "name" : "AGE_FOR_YEAR_ID",
- "datatype" : "smallint"
- }, {
- "id" : "7",
- "name" : "AGE_FOR_QTR_ID",
- "datatype" : "smallint"
- }, {
- "id" : "8",
- "name" : "AGE_FOR_MONTH_ID",
- "datatype" : "smallint"
- }, {
- "id" : "9",
- "name" : "AGE_FOR_WEEK_ID",
- "datatype" : "smallint"
- }, {
- "id" : "10",
- "name" : "AGE_FOR_DT_ID",
- "datatype" : "smallint"
- }, {
- "id" : "11",
- "name" : "AGE_FOR_RTL_YEAR_ID",
- "datatype" : "smallint"
- }, {
- "id" : "12",
- "name" : "AGE_FOR_RTL_QTR_ID",
- "datatype" : "smallint"
- }, {
- "id" : "13",
- "name" : "AGE_FOR_RTL_MONTH_ID",
- "datatype" : "smallint"
- }, {
- "id" : "14",
- "name" : "AGE_FOR_RTL_WEEK_ID",
- "datatype" : "smallint"
- }, {
- "id" : "15",
- "name" : "AGE_FOR_CS_WEEK_ID",
- "datatype" : "smallint"
- }, {
- "id" : "16",
- "name" : "DAY_OF_CAL_ID",
- "datatype" : "int"
- }, {
- "id" : "17",
- "name" : "DAY_OF_YEAR_ID",
- "datatype" : "smallint"
- }, {
- "id" : "18",
- "name" : "DAY_OF_QTR_ID",
- "datatype" : "smallint"
- }, {
- "id" : "19",
- "name" : "DAY_OF_MONTH_ID",
- "datatype" : "smallint"
- }, {
- "id" : "20",
- "name" : "DAY_OF_WEEK_ID",
- "datatype" : "int"
- }, {
- "id" : "21",
- "name" : "WEEK_OF_YEAR_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "22",
- "name" : "WEEK_OF_CAL_ID",
- "datatype" : "int"
- }, {
- "id" : "23",
- "name" : "MONTH_OF_QTR_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "24",
- "name" : "MONTH_OF_YEAR_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "25",
- "name" : "MONTH_OF_CAL_ID",
- "datatype" : "smallint"
- }, {
- "id" : "26",
- "name" : "QTR_OF_YEAR_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "27",
- "name" : "QTR_OF_CAL_ID",
- "datatype" : "smallint"
- }, {
- "id" : "28",
- "name" : "YEAR_OF_CAL_ID",
- "datatype" : "smallint"
- }, {
- "id" : "29",
- "name" : "YEAR_END_DT",
- "datatype" : "string"
- }, {
- "id" : "30",
- "name" : "QTR_END_DT",
- "datatype" : "string"
- }, {
- "id" : "31",
- "name" : "MONTH_END_DT",
- "datatype" : "string"
- }, {
- "id" : "32",
- "name" : "WEEK_END_DT",
- "datatype" : "string"
- }, {
- "id" : "33",
- "name" : "CAL_DT_NAME",
- "datatype" : "string"
- }, {
- "id" : "34",
- "name" : "CAL_DT_DESC",
- "datatype" : "string"
- }, {
- "id" : "35",
- "name" : "CAL_DT_SHORT_NAME",
- "datatype" : "string"
- }, {
- "id" : "36",
- "name" : "YTD_YN_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "37",
- "name" : "QTD_YN_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "38",
- "name" : "MTD_YN_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "39",
- "name" : "WTD_YN_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "40",
- "name" : "SEASON_BEG_DT",
- "datatype" : "string"
- }, {
- "id" : "41",
- "name" : "DAY_IN_YEAR_COUNT",
- "datatype" : "smallint"
- }, {
- "id" : "42",
- "name" : "DAY_IN_QTR_COUNT",
- "datatype" : "tinyint"
- }, {
- "id" : "43",
- "name" : "DAY_IN_MONTH_COUNT",
- "datatype" : "tinyint"
- }, {
- "id" : "44",
- "name" : "DAY_IN_WEEK_COUNT",
- "datatype" : "tinyint"
- }, {
- "id" : "45",
- "name" : "RTL_YEAR_BEG_DT",
- "datatype" : "string"
- }, {
- "id" : "46",
- "name" : "RTL_QTR_BEG_DT",
- "datatype" : "string"
- }, {
- "id" : "47",
- "name" : "RTL_MONTH_BEG_DT",
- "datatype" : "string"
- }, {
- "id" : "48",
- "name" : "RTL_WEEK_BEG_DT",
- "datatype" : "string"
- }, {
- "id" : "49",
- "name" : "CS_WEEK_BEG_DT",
- "datatype" : "string"
- }, {
- "id" : "50",
- "name" : "CAL_DATE",
- "datatype" : "string"
- }, {
- "id" : "51",
- "name" : "DAY_OF_WEEK",
- "datatype" : "string"
- }, {
- "id" : "52",
- "name" : "MONTH_ID",
- "datatype" : "string"
- }, {
- "id" : "53",
- "name" : "PRD_DESC",
- "datatype" : "string"
- }, {
- "id" : "54",
- "name" : "PRD_FLAG",
- "datatype" : "string"
- }, {
- "id" : "55",
- "name" : "PRD_ID",
- "datatype" : "string"
- }, {
- "id" : "56",
- "name" : "PRD_IND",
- "datatype" : "string"
- }, {
- "id" : "57",
- "name" : "QTR_DESC",
- "datatype" : "string"
- }, {
- "id" : "58",
- "name" : "QTR_ID",
- "datatype" : "string"
- }, {
- "id" : "59",
- "name" : "QTR_IND",
- "datatype" : "string"
- }, {
- "id" : "60",
- "name" : "RETAIL_WEEK",
- "datatype" : "string"
- }, {
- "id" : "61",
- "name" : "RETAIL_YEAR",
- "datatype" : "string"
- }, {
- "id" : "62",
- "name" : "RETAIL_START_DATE",
- "datatype" : "string"
- }, {
- "id" : "63",
- "name" : "RETAIL_WK_END_DATE",
- "datatype" : "string"
- }, {
- "id" : "64",
- "name" : "WEEK_IND",
- "datatype" : "string"
- }, {
- "id" : "65",
- "name" : "WEEK_NUM_DESC",
- "datatype" : "string"
- }, {
- "id" : "66",
- "name" : "WEEK_BEG_DATE",
- "datatype" : "string"
- }, {
- "id" : "67",
- "name" : "WEEK_END_DATE",
- "datatype" : "string"
- }, {
- "id" : "68",
- "name" : "WEEK_IN_YEAR_ID",
- "datatype" : "string"
- }, {
- "id" : "69",
- "name" : "WEEK_ID",
- "datatype" : "string"
- }, {
- "id" : "70",
- "name" : "WEEK_BEG_END_DESC_MDY",
- "datatype" : "string"
- }, {
- "id" : "71",
- "name" : "WEEK_BEG_END_DESC_MD",
- "datatype" : "string"
- }, {
- "id" : "72",
- "name" : "YEAR_ID",
- "datatype" : "string"
- }, {
- "id" : "73",
- "name" : "YEAR_IND",
- "datatype" : "string"
- }, {
- "id" : "74",
- "name" : "CAL_DT_MNS_1YEAR_DT",
- "datatype" : "string"
- }, {
- "id" : "75",
- "name" : "CAL_DT_MNS_2YEAR_DT",
- "datatype" : "string"
- }, {
- "id" : "76",
- "name" : "CAL_DT_MNS_1QTR_DT",
- "datatype" : "string"
- }, {
- "id" : "77",
- "name" : "CAL_DT_MNS_2QTR_DT",
- "datatype" : "string"
- }, {
- "id" : "78",
- "name" : "CAL_DT_MNS_1MONTH_DT",
- "datatype" : "string"
- }, {
- "id" : "79",
- "name" : "CAL_DT_MNS_2MONTH_DT",
- "datatype" : "string"
- }, {
- "id" : "80",
- "name" : "CAL_DT_MNS_1WEEK_DT",
- "datatype" : "string"
- }, {
- "id" : "81",
- "name" : "CAL_DT_MNS_2WEEK_DT",
- "datatype" : "string"
- }, {
- "id" : "82",
- "name" : "CURR_CAL_DT_MNS_1YEAR_YN_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "83",
- "name" : "CURR_CAL_DT_MNS_2YEAR_YN_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "84",
- "name" : "CURR_CAL_DT_MNS_1QTR_YN_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "85",
- "name" : "CURR_CAL_DT_MNS_2QTR_YN_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "86",
- "name" : "CURR_CAL_DT_MNS_1MONTH_YN_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "87",
- "name" : "CURR_CAL_DT_MNS_2MONTH_YN_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "88",
- "name" : "CURR_CAL_DT_MNS_1WEEK_YN_IND",
- "datatype" : "tinyint"
- }, {
- "id" : "89",
- "name" : "CURR_CAL_DT_MNS_2WEEK_YN_IND",
- "datatype" : "tinyint"
- }, {
- "id" : "90",
- "name" : "RTL_MONTH_OF_RTL_YEAR_ID",
- "datatype" : "string"
- }, {
- "id" : "91",
- "name" : "RTL_QTR_OF_RTL_YEAR_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "92",
- "name" : "RTL_WEEK_OF_RTL_YEAR_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "93",
- "name" : "SEASON_OF_YEAR_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "94",
- "name" : "YTM_YN_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "95",
- "name" : "YTQ_YN_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "96",
- "name" : "YTW_YN_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "97",
- "name" : "KYLIN_CAL_DT_CRE_DATE",
- "datatype" : "string"
- }, {
- "id" : "98",
- "name" : "KYLIN_CAL_DT_CRE_USER",
- "datatype" : "string"
- }, {
- "id" : "99",
- "name" : "KYLIN_CAL_DT_UPD_DATE",
- "datatype" : "string"
- }, {
- "id" : "100",
- "name" : "KYLIN_CAL_DT_UPD_USER",
- "datatype" : "string"
- } ],
- "database" : "SNOWTEST",
- "last_modified" : 0
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/c6c67197/examples/test_case_data/localmeta/table/SNOWTEST.KYLIN_CATEGORY_GROUPINGS.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/table/SNOWTEST.KYLIN_CATEGORY_GROUPINGS.json b/examples/test_case_data/localmeta/table/SNOWTEST.KYLIN_CATEGORY_GROUPINGS.json
deleted file mode 100644
index c101bd1..0000000
--- a/examples/test_case_data/localmeta/table/SNOWTEST.KYLIN_CATEGORY_GROUPINGS.json
+++ /dev/null
@@ -1,152 +0,0 @@
-{
- "uuid" : "952d11b5-69d9-45d1-92af-227489485e3f",
-
- "name" : "KYLIN_CATEGORY_GROUPINGS",
- "columns" : [ {
- "id" : "1",
- "name" : "LEAF_CATEG_ID",
- "datatype" : "bigint"
- }, {
- "id" : "2",
- "name" : "LEAF_CATEG_NAME",
- "datatype" : "string"
- }, {
- "id" : "3",
- "name" : "SITE_ID",
- "datatype" : "int"
- }, {
- "id" : "4",
- "name" : "CATEG_BUSN_MGR",
- "datatype" : "string"
- }, {
- "id" : "5",
- "name" : "CATEG_BUSN_UNIT",
- "datatype" : "string"
- }, {
- "id" : "6",
- "name" : "REGN_CATEG",
- "datatype" : "string"
- }, {
- "id" : "7",
- "name" : "USER_DEFINED_FIELD1",
- "datatype" : "string"
- }, {
- "id" : "8",
- "name" : "USER_DEFINED_FIELD3",
- "datatype" : "string"
- }, {
- "id" : "9",
- "name" : "KYLIN_GROUPINGS_CRE_DATE",
- "datatype" : "string"
- }, {
- "id" : "10",
- "name" : "KYLIN_GROUPINGS_UPD_DATE",
- "datatype" : "string"
- }, {
- "id" : "11",
- "name" : "KYLIN_GROUPINGS_CRE_USER",
- "datatype" : "string"
- }, {
- "id" : "12",
- "name" : "KYLIN_GROUPINGS_UPD_USER",
- "datatype" : "string"
- }, {
- "id" : "13",
- "name" : "META_CATEG_ID",
- "datatype" : "decimal"
- }, {
- "id" : "14",
- "name" : "META_CATEG_NAME",
- "datatype" : "string"
- }, {
- "id" : "15",
- "name" : "CATEG_LVL2_ID",
- "datatype" : "decimal"
- }, {
- "id" : "16",
- "name" : "CATEG_LVL3_ID",
- "datatype" : "decimal"
- }, {
- "id" : "17",
- "name" : "CATEG_LVL4_ID",
- "datatype" : "decimal"
- }, {
- "id" : "18",
- "name" : "CATEG_LVL5_ID",
- "datatype" : "decimal"
- }, {
- "id" : "19",
- "name" : "CATEG_LVL6_ID",
- "datatype" : "decimal"
- }, {
- "id" : "20",
- "name" : "CATEG_LVL7_ID",
- "datatype" : "decimal"
- }, {
- "id" : "21",
- "name" : "CATEG_LVL2_NAME",
- "datatype" : "string"
- }, {
- "id" : "22",
- "name" : "CATEG_LVL3_NAME",
- "datatype" : "string"
- }, {
- "id" : "23",
- "name" : "CATEG_LVL4_NAME",
- "datatype" : "string"
- }, {
- "id" : "24",
- "name" : "CATEG_LVL5_NAME",
- "datatype" : "string"
- }, {
- "id" : "25",
- "name" : "CATEG_LVL6_NAME",
- "datatype" : "string"
- }, {
- "id" : "26",
- "name" : "CATEG_LVL7_NAME",
- "datatype" : "string"
- }, {
- "id" : "27",
- "name" : "CATEG_FLAGS",
- "datatype" : "decimal"
- }, {
- "id" : "28",
- "name" : "ADULT_CATEG_YN",
- "datatype" : "string"
- }, {
- "id" : "29",
- "name" : "DOMAIN_ID",
- "datatype" : "decimal"
- }, {
- "id" : "30",
- "name" : "USER_DEFINED_FIELD5",
- "datatype" : "string"
- }, {
- "id" : "31",
- "name" : "VCS_ID",
- "datatype" : "decimal"
- }, {
- "id" : "32",
- "name" : "GCS_ID",
- "datatype" : "decimal"
- }, {
- "id" : "33",
- "name" : "MOVE_TO",
- "datatype" : "decimal"
- }, {
- "id" : "34",
- "name" : "SAP_CATEGORY_ID",
- "datatype" : "decimal"
- }, {
- "id" : "35",
- "name" : "SRC_ID",
- "datatype" : "tinyint"
- }, {
- "id" : "36",
- "name" : "BSNS_VRTCL_NAME",
- "datatype" : "string"
- } ],
- "database" : "SNOWTEST",
- "last_modified" : 0
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/c6c67197/examples/test_case_data/localmeta/table/SNOWTEST.KYLIN_COUNTRY.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/table/SNOWTEST.KYLIN_COUNTRY.json b/examples/test_case_data/localmeta/table/SNOWTEST.KYLIN_COUNTRY.json
deleted file mode 100644
index 2f4d723..0000000
--- a/examples/test_case_data/localmeta/table/SNOWTEST.KYLIN_COUNTRY.json
+++ /dev/null
@@ -1,24 +0,0 @@
-{
- "uuid" : "e286e39e-40d7-44c2-8fa2-41b365632882",
-
- "name" : "KYLIN_COUNTRY",
- "columns" : [ {
- "id" : "1",
- "name" : "COUNTRY",
- "datatype" : "string"
- }, {
- "id" : "2",
- "name" : "LATITUDE",
- "datatype" : "double"
- }, {
- "id" : "3",
- "name" : "LONGITUDE",
- "datatype" : "double"
- }, {
- "id" : "4",
- "name" : "NAME",
- "datatype" : "string"
- } ],
- "database" : "SNOWTEST",
- "last_modified" : 0
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/c6c67197/examples/test_case_data/localmeta/table/SNOWTEST.KYLIN_SALES.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/table/SNOWTEST.KYLIN_SALES.json b/examples/test_case_data/localmeta/table/SNOWTEST.KYLIN_SALES.json
deleted file mode 100644
index a9dcc08..0000000
--- a/examples/test_case_data/localmeta/table/SNOWTEST.KYLIN_SALES.json
+++ /dev/null
@@ -1,56 +0,0 @@
-{
- "uuid" : "e286e39e-40d7-44c2-8fa2-41b365522771",
-
- "name" : "KYLIN_SALES",
- "columns" : [ {
- "id" : "1",
- "name" : "TRANS_ID",
- "datatype" : "bigint"
- }, {
- "id" : "2",
- "name" : "PART_DT",
- "datatype" : "date"
- }, {
- "id" : "3",
- "name" : "LSTG_FORMAT_NAME",
- "datatype" : "string"
- }, {
- "id" : "4",
- "name" : "LEAF_CATEG_ID",
- "datatype" : "bigint"
- }, {
- "id" : "5",
- "name" : "LSTG_SITE_ID",
- "datatype" : "int"
- }, {
- "id" : "6",
- "name" : "SLR_SEGMENT_CD",
- "datatype" : "smallint"
- }, {
- "id" : "7",
- "name" : "PRICE",
- "datatype" : "decimal(19,4)"
- }, {
- "id" : "8",
- "name" : "ITEM_COUNT",
- "datatype" : "bigint"
- }, {
- "id" : "9",
- "name" : "SELLER_ID",
- "datatype" : "bigint"
- }, {
- "id" : "10",
- "name" : "BUYER_ID",
- "datatype" : "bigint"
- }, {
- "id" : "11",
- "name" : "OPS_USER_ID",
- "datatype" : "string"
- }, {
- "id" : "12",
- "name" : "OPS_REGION",
- "datatype" : "string"
- } ],
- "database" : "SNOWTEST",
- "last_modified" : 0
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/c6c67197/kylin-it/src/test/java/org/apache/kylin/query/H2Database.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/query/H2Database.java b/kylin-it/src/test/java/org/apache/kylin/query/H2Database.java
index 3966c3a..c449837 100644
--- a/kylin-it/src/test/java/org/apache/kylin/query/H2Database.java
+++ b/kylin-it/src/test/java/org/apache/kylin/query/H2Database.java
@@ -40,7 +40,16 @@ public class H2Database {
@SuppressWarnings("unused")
private static final Logger logger = LoggerFactory.getLogger(H2Database.class);
- private static final String[] ALL_TABLES = new String[] { "edw.test_cal_dt", "default.test_category_groupings", "default.test_kylin_fact", "default.test_order", "edw.test_seller_type_dim", "edw.test_sites", "default.streaming_table" };
+ private static final String[] ALL_TABLES = new String[] { //
+ "edw.test_cal_dt", //
+ "default.test_category_groupings", //
+ "default.test_kylin_fact", //
+ "default.test_order", //
+ "edw.test_seller_type_dim", //
+ "edw.test_sites", //
+ "default.test_account", //
+ "default.test_country", //
+ "default.streaming_table" };
private static final Map<String, String> javaToH2DataTypeMapping = new HashMap<String, String>();
static {
[3/7] kylin git commit: KYLIN-2306 fall back to abstract when
NoClassFoundException
Posted by li...@apache.org.
KYLIN-2306 fall back to abstract when NoClassFoundException
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/73a40296
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/73a40296
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/73a40296
Branch: refs/heads/KYLIN-2314
Commit: 73a40296e33921a000cb97faa3321dd53b8dbf41
Parents: 488283d
Author: Billy Liu <bi...@apache.org>
Authored: Fri Dec 23 23:49:02 2016 +0800
Committer: Billy Liu <bi...@apache.org>
Committed: Fri Dec 23 23:49:20 2016 +0800
----------------------------------------------------------------------
.../kylin/job/execution/ExecutableManager.java | 59 ++++++++++++--------
1 file changed, 35 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/73a40296/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java
----------------------------------------------------------------------
diff --git a/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java b/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java
index 0c49a3e..0273fd8 100644
--- a/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java
+++ b/core-job/src/main/java/org/apache/kylin/job/execution/ExecutableManager.java
@@ -44,10 +44,16 @@ public class ExecutableManager {
private static final Logger logger = LoggerFactory.getLogger(ExecutableManager.class);
private static final ConcurrentHashMap<KylinConfig, ExecutableManager> CACHE = new ConcurrentHashMap<KylinConfig, ExecutableManager>();
-
+
private final KylinConfig config;
private final ExecutableDao executableDao;
+ private ExecutableManager(KylinConfig config) {
+ logger.info("Using metadata url: " + config);
+ this.config = config;
+ this.executableDao = ExecutableDao.getInstance(config);
+ }
+
public static ExecutableManager getInstance(KylinConfig config) {
ExecutableManager r = CACHE.get(config);
if (r == null) {
@@ -69,10 +75,20 @@ public class ExecutableManager {
CACHE.clear();
}
- private ExecutableManager(KylinConfig config) {
- logger.info("Using metadata url: " + config);
- this.config = config;
- this.executableDao = ExecutableDao.getInstance(config);
+ private static ExecutablePO parse(AbstractExecutable executable) {
+ ExecutablePO result = new ExecutablePO();
+ result.setName(executable.getName());
+ result.setUuid(executable.getId());
+ result.setType(executable.getClass().getName());
+ result.setParams(executable.getParams());
+ if (executable instanceof ChainedExecutable) {
+ List<ExecutablePO> tasks = Lists.newArrayList();
+ for (AbstractExecutable task : ((ChainedExecutable) executable).getTasks()) {
+ tasks.add(parse(task));
+ }
+ result.setTasks(tasks);
+ }
+ return result;
}
public void addJob(AbstractExecutable executable) {
@@ -321,7 +337,6 @@ public class ExecutableManager {
updateJobOutput(jobId, ExecutableState.DISCARDED, null, null);
}
-
public void rollbackJob(String jobId, String stepId) {
AbstractExecutable job = getJob(jobId);
if (job == null) {
@@ -409,22 +424,6 @@ public class ExecutableManager {
addJobInfo(id, info);
}
- private static ExecutablePO parse(AbstractExecutable executable) {
- ExecutablePO result = new ExecutablePO();
- result.setName(executable.getName());
- result.setUuid(executable.getId());
- result.setType(executable.getClass().getName());
- result.setParams(executable.getParams());
- if (executable instanceof ChainedExecutable) {
- List<ExecutablePO> tasks = Lists.newArrayList();
- for (AbstractExecutable task : ((ChainedExecutable) executable).getTasks()) {
- tasks.add(parse(task));
- }
- result.setTasks(tasks);
- }
- return result;
- }
-
private AbstractExecutable parseTo(ExecutablePO executablePO) {
if (executablePO == null) {
logger.warn("executablePO is null");
@@ -457,8 +456,14 @@ public class ExecutableManager {
logger.warn("executablePO is null");
return null;
}
+ String type = executablePO.getType();
try {
- Class<? extends AbstractExecutable> clazz = ClassUtil.forName(expectedClass.getName(), AbstractExecutable.class);
+ Class<? extends AbstractExecutable> clazz = null;
+ try {
+ clazz = ClassUtil.forName(type, AbstractExecutable.class);
+ } catch (ClassNotFoundException e) {
+ clazz = ClassUtil.forName(expectedClass.getName(), AbstractExecutable.class);
+ }
Constructor<? extends AbstractExecutable> constructor = clazz.getConstructor();
AbstractExecutable result = constructor.newInstance();
result.initConfig(config);
@@ -469,7 +474,13 @@ public class ExecutableManager {
if (tasks != null && !tasks.isEmpty()) {
Preconditions.checkArgument(result instanceof ChainedExecutable);
for (ExecutablePO subTask : tasks) {
- ((ChainedExecutable) result).addTask(parseToAbstract(subTask, DefaultChainedExecutable.class));
+ AbstractExecutable parseToTask = null;
+ try {
+ parseToTask = parseTo(subTask);
+ } catch (IllegalStateException e) {
+ parseToTask = parseToAbstract(subTask, DefaultChainedExecutable.class);
+ }
+ ((ChainedExecutable) result).addTask(parseToTask);
}
}
return result;
[7/7] kylin git commit: minor, add time log for BuildCubeWithStream
Posted by li...@apache.org.
minor, add time log for BuildCubeWithStream
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4881e282
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4881e282
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4881e282
Branch: refs/heads/KYLIN-2314
Commit: 4881e282eac3d836fb5ab3effca46f8a3808da41
Parents: c6c6719
Author: Yang Li <li...@apache.org>
Authored: Sat Dec 24 15:34:48 2016 +0800
Committer: Yang Li <li...@apache.org>
Committed: Sat Dec 24 15:34:48 2016 +0800
----------------------------------------------------------------------
.../org/apache/kylin/provision/BuildCubeWithStream.java | 10 ++++++++--
1 file changed, 8 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/4881e282/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
index 8abb84c..194ea35 100644
--- a/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
+++ b/kylin-it/src/test/java/org/apache/kylin/provision/BuildCubeWithStream.java
@@ -311,6 +311,9 @@ public class BuildCubeWithStream {
}
public static void main(String[] args) throws Exception {
+ long start = System.currentTimeMillis();
+ int exitCode = 0;
+
BuildCubeWithStream buildCubeWithStream = null;
try {
beforeClass();
@@ -320,11 +323,14 @@ public class BuildCubeWithStream {
logger.info("Build is done");
buildCubeWithStream.cleanup();
logger.info("Going to exit");
- System.exit(0);
} catch (Throwable e) {
logger.error("error", e);
- System.exit(1);
+ exitCode = 1;
}
+ long millis = System.currentTimeMillis() - start;
+ System.out.println("Time elapsed: " + (millis / 1000) + " sec - in " + BuildCubeWithStream.class.getName());
+
+ System.exit(exitCode);
}
}
[6/7] kylin git commit: KYLIN-2314 Use col identity instead of col
name in dictionary refs
Posted by li...@apache.org.
KYLIN-2314 Use col identity instead of col name in dictionary refs
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c6c67197
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c6c67197
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c6c67197
Branch: refs/heads/KYLIN-2314
Commit: c6c6719730e05c66c14407cfd7926a0a54a6009a
Parents: 30d3763
Author: Li Yang <li...@apache.org>
Authored: Fri Dec 23 18:06:58 2016 +0800
Committer: Yang Li <li...@apache.org>
Committed: Sat Dec 24 14:46:39 2016 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/job/DeployUtil.java | 26 +-
.../java/org/apache/kylin/cube/CubeManager.java | 2 +-
.../java/org/apache/kylin/cube/CubeSegment.java | 30 +-
.../org/apache/kylin/cube/CubeDescTest.java | 40 +-
.../kylin/metadata/MetadataManagerTest.java | 67 ++-
.../kylin/metadata/model/JoinsTreeTest.java | 11 +-
.../kylin/engine/mr/JobBuilderSupport.java | 2 -
.../engine/mr/steps/CreateDictionaryJob.java | 5 +-
.../engine/mr/steps/CubingExecutableUtil.java | 8 -
.../engine/mr/steps/FactDistinctColumnsJob.java | 3 +
.../mr/steps/FactDistinctColumnsReducer.java | 39 +-
.../mr/steps/FactDistinctHiveColumnsMapper.java | 81 ++--
.../mr/steps/UpdateCubeInfoAfterBuildStep.java | 4 +-
.../mr/steps/UpdateCubeInfoAfterMergeStep.java | 1 -
.../localmeta/cube_desc/ci_inner_join_cube.json | 248 ++++++++---
.../localmeta/cube_desc/ci_left_join_cube.json | 248 ++++++++---
.../test_kylin_snowflake_sales_cube.json | 268 ------------
.../localmeta/data/DEFAULT.TEST_COUNTRY.csv | 244 +++++++++++
.../model_desc/ci_inner_join_model.json | 100 ++++-
.../model_desc/ci_left_join_model.json | 100 ++++-
.../test_kylin_snowflake_sales_model.json | 87 ----
.../localmeta/table/DEFAULT.TEST_ACCOUNT.json | 34 ++
.../localmeta/table/DEFAULT.TEST_COUNTRY.json | 24 ++
.../table/DEFAULT.TEST_KYLIN_COUNTRY.json | 24 --
.../localmeta/table/SNOWTEST.KYLIN_ACCOUNT.json | 28 --
.../localmeta/table/SNOWTEST.KYLIN_CAL_DT.json | 408 -------------------
.../SNOWTEST.KYLIN_CATEGORY_GROUPINGS.json | 152 -------
.../localmeta/table/SNOWTEST.KYLIN_COUNTRY.json | 24 --
.../localmeta/table/SNOWTEST.KYLIN_SALES.json | 56 ---
.../java/org/apache/kylin/query/H2Database.java | 11 +-
30 files changed, 1080 insertions(+), 1295 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/c6c67197/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
----------------------------------------------------------------------
diff --git a/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java b/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
index 564a4e2..089c3ed 100644
--- a/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
+++ b/assembly/src/test/java/org/apache/kylin/job/DeployUtil.java
@@ -125,11 +125,15 @@ public class DeployUtil {
static final String TABLE_CATEGORY_GROUPINGS = "default.test_category_groupings";
static final String TABLE_KYLIN_FACT = "default.test_kylin_fact";
static final String TABLE_ORDER = "default.test_order";
+ static final String TABLE_ACCOUNT = "default.test_account";
+ static final String TABLE_COUNTRY = "default.test_country";
static final String VIEW_SELLER_TYPE_DIM = "edw.test_seller_type_dim";
static final String TABLE_SELLER_TYPE_DIM_TABLE = "edw.test_seller_type_dim_table";
static final String TABLE_SITES = "edw.test_sites";
- static final String[] TABLE_NAMES = new String[] { TABLE_CAL_DT, TABLE_ORDER, TABLE_CATEGORY_GROUPINGS, TABLE_KYLIN_FACT, TABLE_SELLER_TYPE_DIM_TABLE, TABLE_SITES };
+ static final String[] TABLE_NAMES = new String[] { //
+ TABLE_CAL_DT, TABLE_ORDER, TABLE_CATEGORY_GROUPINGS, TABLE_KYLIN_FACT, //
+ TABLE_SELLER_TYPE_DIM_TABLE, TABLE_SITES, TABLE_ACCOUNT, TABLE_COUNTRY };
public static void prepareTestDataForNormalCubes(String modelName) throws Exception {
@@ -228,21 +232,15 @@ public class DeployUtil {
IHiveClient hiveClient = HiveClientFactory.getHiveClient();
// create hive tables
hiveClient.executeHQL("CREATE DATABASE IF NOT EXISTS EDW");
- hiveClient.executeHQL(generateCreateTableHql(metaMgr.getTableDesc(TABLE_CAL_DT.toUpperCase())));
- hiveClient.executeHQL(generateCreateTableHql(metaMgr.getTableDesc(TABLE_CATEGORY_GROUPINGS.toUpperCase())));
- hiveClient.executeHQL(generateCreateTableHql(metaMgr.getTableDesc(TABLE_KYLIN_FACT.toUpperCase())));
- hiveClient.executeHQL(generateCreateTableHql(metaMgr.getTableDesc(TABLE_ORDER.toUpperCase())));
- hiveClient.executeHQL(generateCreateTableHql(metaMgr.getTableDesc(TABLE_SELLER_TYPE_DIM_TABLE.toUpperCase())));
- hiveClient.executeHQL(generateCreateTableHql(metaMgr.getTableDesc(TABLE_SITES.toUpperCase())));
+ for (String tablename : TABLE_NAMES) {
+ hiveClient.executeHQL(generateCreateTableHql(metaMgr.getTableDesc(tablename.toUpperCase())));
+ }
// load data to hive tables
// LOAD DATA LOCAL INPATH 'filepath' [OVERWRITE] INTO TABLE tablename
- hiveClient.executeHQL(generateLoadDataHql(TABLE_CAL_DT, tableFileDir));
- hiveClient.executeHQL(generateLoadDataHql(TABLE_CATEGORY_GROUPINGS, tableFileDir));
- hiveClient.executeHQL(generateLoadDataHql(TABLE_KYLIN_FACT, tableFileDir));
- hiveClient.executeHQL(generateLoadDataHql(TABLE_ORDER, tableFileDir));
- hiveClient.executeHQL(generateLoadDataHql(TABLE_SELLER_TYPE_DIM_TABLE, tableFileDir));
- hiveClient.executeHQL(generateLoadDataHql(TABLE_SITES, tableFileDir));
+ for (String tablename : TABLE_NAMES) {
+ hiveClient.executeHQL(generateLoadDataHql(tablename.toUpperCase(), tableFileDir));
+ }
final HiveCmdBuilder hiveCmdBuilder = new HiveCmdBuilder();
hiveCmdBuilder.addStatements(generateCreateViewHql(VIEW_SELLER_TYPE_DIM, TABLE_SELLER_TYPE_DIM_TABLE));
@@ -251,7 +249,7 @@ public class DeployUtil {
}
private static String generateLoadDataHql(String tableName, String tableFileDir) {
- return "LOAD DATA LOCAL INPATH '" + tableFileDir + "/" + tableName.toUpperCase() + ".csv' OVERWRITE INTO TABLE " + tableName.toUpperCase();
+ return "LOAD DATA LOCAL INPATH '" + tableFileDir + "/" + tableName + ".csv' OVERWRITE INTO TABLE " + tableName;
}
private static String[] generateCreateTableHql(TableDesc tableDesc) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/c6c67197/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index fe2030a..9670b89 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -240,7 +240,7 @@ public class CubeManager implements IRealizationProvider {
if (dictInfo != null) {
Dictionary<?> dict = dictInfo.getDictionaryObject();
cubeSeg.putDictResPath(col, dictInfo.getResourcePath());
- cubeSeg.getRowkeyStats().add(new Object[] { col.getName(), dict.getSize(), dict.getSizeOfId() });
+ cubeSeg.getRowkeyStats().add(new Object[] { col.getIdentity(), dict.getSize(), dict.getSizeOfId() });
CubeUpdate update = new CubeUpdate(cubeSeg.getCubeInstance());
update.setToUpdateSegs(cubeSeg);
http://git-wip-us.apache.org/repos/asf/kylin/blob/c6c67197/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
index e155f86..36a6044 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
@@ -98,9 +98,6 @@ public class CubeSegment implements Comparable<CubeSegment>, IBuildable, ISegmen
@JsonProperty("snapshots")
private ConcurrentHashMap<String, String> snapshots; // table name ==> snapshot resource path
- @JsonProperty("index_path")
- private String indexPath;
-
@JsonProperty("rowkey_stats")
private List<Object[]> rowkeyStats = Lists.newArrayList();
@@ -296,15 +293,22 @@ public class CubeSegment implements Comparable<CubeSegment>, IBuildable, ISegmen
}
public String getDictResPath(TblColRef col) {
- return getDictionaries().get(dictKey(col));
+ String r;
+ String dictKey = col.getIdentity();
+ r = getDictionaries().get(dictKey);
+
+ // try Kylin v1.x dict key as well
+ if (r == null) {
+ String v1DictKey = col.getTable() + "/" + col.getName();
+ r = getDictionaries().get(v1DictKey);
+ }
+
+ return r;
}
public void putDictResPath(TblColRef col, String dictResPath) {
- getDictionaries().put(dictKey(col), dictResPath);
- }
-
- private String dictKey(TblColRef col) {
- return col.getTable() + "/" + col.getName();
+ String dictKey = col.getIdentity();
+ getDictionaries().put(dictKey, dictResPath);
}
public void setStorageLocationIdentifier(String storageLocationIdentifier) {
@@ -523,14 +527,6 @@ public class CubeSegment implements Comparable<CubeSegment>, IBuildable, ISegmen
return cubeInstance;
}
- public String getIndexPath() {
- return indexPath;
- }
-
- public void setIndexPath(String indexPath) {
- this.indexPath = indexPath;
- }
-
public Map<String, String> getAdditionalInfo() {
return additionalInfo;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/c6c67197/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java b/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java
index fb003bc..39b6433 100644
--- a/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/cube/CubeDescTest.java
@@ -30,7 +30,9 @@ import org.apache.kylin.common.util.JsonUtil;
import org.apache.kylin.common.util.LocalFileMetadataTestCase;
import org.apache.kylin.cube.model.AggregationGroup;
import org.apache.kylin.cube.model.CubeDesc;
+import org.apache.kylin.cube.model.DimensionDesc;
import org.apache.kylin.cube.model.SelectRule;
+import org.apache.kylin.metadata.model.MeasureDesc;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@@ -40,8 +42,7 @@ import org.junit.rules.ExpectedException;
import com.google.common.collect.Maps;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.*;
/**
* @author yangli9
@@ -94,8 +95,39 @@ public class CubeDescTest extends LocalFileMetadataTestCase {
@Test
public void testCiCube() {
CubeDescManager mgr = CubeDescManager.getInstance(getTestConfig());
- assertNotNull(mgr.getCubeDesc("ci_left_join_cube"));
- assertNotNull(mgr.getCubeDesc("ci_inner_join_cube"));
+ CubeDesc lc = mgr.getCubeDesc("ci_left_join_cube");
+ CubeDesc ic = mgr.getCubeDesc("ci_inner_join_cube");
+ assertNotNull(lc);
+ assertNotNull(ic);
+
+ // assert the two CI cubes are identical apart from the left/inner difference
+ assertEquals(lc.getDimensions().size(), ic.getDimensions().size());
+ for (int i = 0, n = lc.getDimensions().size(); i < n; i++) {
+ DimensionDesc ld = lc.getDimensions().get(i);
+ DimensionDesc id = ic.getDimensions().get(i);
+ assertEquals(ld.getTable(), id.getTable());
+ assertEquals(ld.getColumn(), id.getColumn());
+ assertArrayEquals(ld.getDerived(), id.getDerived());
+ }
+
+ assertEquals(lc.getMeasures().size(), ic.getMeasures().size());
+ for (int i = 0, n = lc.getMeasures().size(); i < n; i++) {
+ MeasureDesc lm = lc.getMeasures().get(i);
+ MeasureDesc im = ic.getMeasures().get(i);
+ assertEquals(lm.getName(), im.getName());
+ assertEquals(lm.getFunction().getFullExpression(), im.getFunction().getFullExpression());
+ assertEquals(lm.getFunction().getReturnType(), im.getFunction().getReturnType());
+ }
+
+ assertEquals(lc.getAggregationGroups().size(), ic.getAggregationGroups().size());
+ for (int i = 0, n = lc.getAggregationGroups().size(); i < n; i++) {
+ AggregationGroup lag = lc.getAggregationGroups().get(i);
+ AggregationGroup iag = ic.getAggregationGroups().get(i);
+ assertArrayEquals(lag.getIncludes(), iag.getIncludes());
+ assertArrayEquals(lag.getSelectRule().mandatory_dims, iag.getSelectRule().mandatory_dims);
+ assertArrayEquals(lag.getSelectRule().hierarchy_dims, iag.getSelectRule().hierarchy_dims);
+ assertArrayEquals(lag.getSelectRule().joint_dims, iag.getSelectRule().joint_dims);
+ }
}
@Test
http://git-wip-us.apache.org/repos/asf/kylin/blob/c6c67197/core-metadata/src/test/java/org/apache/kylin/metadata/MetadataManagerTest.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/test/java/org/apache/kylin/metadata/MetadataManagerTest.java b/core-metadata/src/test/java/org/apache/kylin/metadata/MetadataManagerTest.java
index a27f4cf..b372be2 100644
--- a/core-metadata/src/test/java/org/apache/kylin/metadata/MetadataManagerTest.java
+++ b/core-metadata/src/test/java/org/apache/kylin/metadata/MetadataManagerTest.java
@@ -19,6 +19,7 @@
package org.apache.kylin.metadata;
import static org.apache.kylin.metadata.MetadataManager.getInstance;
+import static org.junit.Assert.*;
import java.io.IOException;
import java.util.ArrayList;
@@ -26,6 +27,8 @@ import java.util.List;
import org.apache.kylin.common.util.LocalFileMetadataTestCase;
import org.apache.kylin.metadata.model.DataModelDesc;
+import org.apache.kylin.metadata.model.JoinTableDesc;
+import org.apache.kylin.metadata.model.ModelDimensionDesc;
import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.metadata.model.TableExtDesc;
import org.junit.After;
@@ -50,8 +53,50 @@ public class MetadataManagerTest extends LocalFileMetadataTestCase {
@Test
public void testCiModel() {
MetadataManager mgr = getInstance(getTestConfig());
- Assert.assertNotNull(mgr.getDataModelDesc("ci_left_join_model"));
- Assert.assertNotNull(mgr.getDataModelDesc("ci_inner_join_model"));
+ DataModelDesc lm = mgr.getDataModelDesc("ci_left_join_model");
+ DataModelDesc im = mgr.getDataModelDesc("ci_inner_join_model");
+ assertSnowflakeQuality(lm);
+ assertSnowflakeQuality(im);
+
+ // check inner/left models are identical apart from the left/inner difference
+ assertEquals(lm.getJoinTables().length, im.getJoinTables().length);
+ for (int i = 0, n = im.getJoinTables().length; i < n; i++) {
+ JoinTableDesc lt = lm.getJoinTables()[i];
+ JoinTableDesc it = im.getJoinTables()[i];
+ assertEquals(lt.getAlias(), it.getAlias());
+ assertEquals(lt.getKind(), it.getKind());
+ assertEquals(lt.getTable(), it.getTable());
+ assertArrayEquals(lt.getJoin().getForeignKey(), it.getJoin().getForeignKey());
+ assertArrayEquals(lt.getJoin().getPrimaryKey(), it.getJoin().getPrimaryKey());
+ assertTrue(lt.getJoin().isLeftJoin());
+ assertTrue(it.getJoin().isInnerJoin());
+ }
+
+ assertEquals(lm.getDimensions().size(), im.getDimensions().size());
+ for (int i = 0, n = im.getDimensions().size(); i < n; i++) {
+ ModelDimensionDesc ld = lm.getDimensions().get(i);
+ ModelDimensionDesc id = im.getDimensions().get(i);
+ assertEquals(ld.getTable(), id.getTable());
+ assertArrayEquals(ld.getColumns(), id.getColumns());
+ }
+
+ assertArrayEquals(lm.getMetrics(), im.getMetrics());
+ }
+
+ private void assertSnowflakeQuality(DataModelDesc model) {
+ Assert.assertNotNull(model);
+ try {
+ model.findTable("TEST_COUNTRY");
+ Assert.fail();
+ } catch (IllegalArgumentException ex) {
+ // excepted
+ }
+
+ Assert.assertNotNull(model.findTable("BUYER_COUNTRY"));
+ Assert.assertNotNull(model.findTable("SELLER_COUNTRY"));
+ Assert.assertNotNull(model.findColumn("BUYER_COUNTRY.NAME"));
+ Assert.assertNotNull(model.findColumn("BUYER_ID"));
+
}
@Test
@@ -82,24 +127,6 @@ public class MetadataManagerTest extends LocalFileMetadataTestCase {
}
@Test
- public void testSnowflakeDataModel() throws Exception {
- DataModelDesc model = getInstance(getTestConfig()).getDataModelDesc("test_kylin_snowflake_sales_model");
- Assert.assertTrue(model.getDimensions().size() > 0);
-
- try {
- model.findTable("KYLIN_COUNTRY");
- Assert.fail();
- } catch (IllegalArgumentException ex) {
- // excepted
- }
-
- Assert.assertNotNull(model.findTable("BUYER_COUNTRY"));
- Assert.assertNotNull(model.findTable("SELLER_COUNTRY"));
- Assert.assertNotNull(model.findColumn("BUYER_COUNTRY.NAME"));
- Assert.assertNotNull(model.findColumn("BUYER_ID"));
- }
-
- @Test
public void testTableSample() throws IOException {
TableExtDesc tableExtDesc = getInstance(getTestConfig()).getTableExt("TEST.TEST_TABLE");
Assert.assertNotNull(tableExtDesc);
http://git-wip-us.apache.org/repos/asf/kylin/blob/c6c67197/core-metadata/src/test/java/org/apache/kylin/metadata/model/JoinsTreeTest.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/test/java/org/apache/kylin/metadata/model/JoinsTreeTest.java b/core-metadata/src/test/java/org/apache/kylin/metadata/model/JoinsTreeTest.java
index f52ef45..d5e9de7 100644
--- a/core-metadata/src/test/java/org/apache/kylin/metadata/model/JoinsTreeTest.java
+++ b/core-metadata/src/test/java/org/apache/kylin/metadata/model/JoinsTreeTest.java
@@ -48,21 +48,22 @@ public class JoinsTreeTest extends LocalFileMetadataTestCase {
@Test
public void testBasics() {
MetadataManager mgr = MetadataManager.getInstance(KylinConfig.getInstanceFromEnv());
- DataModelDesc model = mgr.getDataModelDesc("test_kylin_snowflake_sales_model");
+ DataModelDesc model = mgr.getDataModelDesc("ci_left_join_model");
JoinsTree joinsTree = model.getJoinsTree();
Chain chain = joinsTree.tableChains.get("BUYER_COUNTRY");
assertTrue(chain.table == model.findTable("BUYER_COUNTRY"));
assertTrue(chain.fkSide.table == model.findTable("BUYER_ACCOUNT"));
- assertTrue(chain.fkSide.fkSide.table == model.findTable("KYLIN_SALES"));
- assertTrue(chain.fkSide.fkSide.join == null);
- assertTrue(chain.fkSide.fkSide.fkSide == null);
+ assertTrue(chain.fkSide.fkSide.table == model.findTable("TEST_ORDER"));
+ assertTrue(chain.fkSide.fkSide.fkSide.table == model.findTable("TEST_KYLIN_FACT"));
+ assertTrue(chain.fkSide.fkSide.fkSide.join == null);
+ assertTrue(chain.fkSide.fkSide.fkSide.fkSide == null);
}
@Test
public void testMatch() {
MetadataManager mgr = MetadataManager.getInstance(KylinConfig.getInstanceFromEnv());
- DataModelDesc model = mgr.getDataModelDesc("test_kylin_snowflake_sales_model");
+ DataModelDesc model = mgr.getDataModelDesc("ci_inner_join_model");
JoinsTree joinsTree = model.getJoinsTree();
Map<String, String> matchUp = joinsTree.matches(joinsTree);
http://git-wip-us.apache.org/repos/asf/kylin/blob/c6c67197/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
index 47eb9c3..5f5814b 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java
@@ -100,7 +100,6 @@ public class JobBuilderSupport {
CubingExecutableUtil.setCubeName(seg.getRealization().getName(), result.getParams());
CubingExecutableUtil.setSegmentId(seg.getUuid(), result.getParams());
CubingExecutableUtil.setCubingJobId(jobId, result.getParams());
- CubingExecutableUtil.setIndexPath(this.getSecondaryIndexPath(jobId), result.getParams());
return result;
@@ -125,7 +124,6 @@ public class JobBuilderSupport {
CubingExecutableUtil.setSegmentId(seg.getUuid(), result.getParams());
CubingExecutableUtil.setCubingJobId(jobId, result.getParams());
CubingExecutableUtil.setMergingSegmentIds(mergingSegmentIds, result.getParams());
- CubingExecutableUtil.setIndexPath(this.getSecondaryIndexPath(jobId), result.getParams());
return result;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/c6c67197/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CreateDictionaryJob.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CreateDictionaryJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CreateDictionaryJob.java
index 4985503..8b9697e 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CreateDictionaryJob.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CreateDictionaryJob.java
@@ -57,14 +57,13 @@ public class CreateDictionaryJob extends AbstractHadoopJob {
DictionaryGeneratorCLI.processSegment(config, cubeName, segmentID, new DistinctColumnValuesProvider() {
@Override
public ReadableTable getDistinctValuesFor(TblColRef col) {
- return new SortedColumnDFSFile(factColumnsInputPath + "/" + col.getName(), col.getType());
+ return new SortedColumnDFSFile(factColumnsInputPath + "/" + col.getIdentity(), col.getType());
}
}, new DictionaryProvider() {
@Override
public Dictionary<String> getDictionary(TblColRef col) throws IOException {
- Path colDir = new Path(factColumnsInputPath, col.getName());
- Path dictFile = new Path(colDir, col.getName() + FactDistinctColumnsReducer.DICT_FILE_POSTFIX);
+ Path dictFile = new Path(factColumnsInputPath, col.getIdentity() + FactDistinctColumnsReducer.DICT_FILE_POSTFIX);
FileSystem fs = HadoopUtil.getFileSystem(dictFile.toString());
if (fs.exists(dictFile) == false)
return null;
http://git-wip-us.apache.org/repos/asf/kylin/blob/c6c67197/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CubingExecutableUtil.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CubingExecutableUtil.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CubingExecutableUtil.java
index b0d5a89..65c5869 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CubingExecutableUtil.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/CubingExecutableUtil.java
@@ -135,12 +135,4 @@ public class CubingExecutableUtil {
return params.get(MERGED_STATISTICS_PATH);
}
- public static void setIndexPath(String indexPath, Map<String, String> params) {
- params.put(INDEX_PATH, indexPath);
- }
-
- public static String getIndexPath(Map<String, String> params) {
- return params.get(INDEX_PATH);
- }
-
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/c6c67197/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsJob.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsJob.java
index 2eb694e..9fc8922 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsJob.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsJob.java
@@ -145,6 +145,9 @@ public class FactDistinctColumnsJob extends AbstractHadoopJob {
job.setPartitionerClass(FactDistinctColumnPartitioner.class);
job.setNumReduceTasks(numberOfReducers);
+ // important, reducer writes HDFS directly at the moment
+ job.setReduceSpeculativeExecution(false);
+
FileOutputFormat.setOutputPath(job, output);
job.getConfiguration().set(BatchConstants.CFG_OUTPUT_PATH, output.toString());
http://git-wip-us.apache.org/repos/asf/kylin/blob/c6c67197/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
index 0223914..5d42797 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
@@ -59,7 +59,7 @@ import com.google.common.collect.Maps;
*/
public class FactDistinctColumnsReducer extends KylinReducer<SelfDefineSortableKey, Text, NullWritable, Text> {
- protected static final Logger logger = LoggerFactory.getLogger(FactDistinctColumnsReducer.class);
+ private static final Logger logger = LoggerFactory.getLogger(FactDistinctColumnsReducer.class);
private List<TblColRef> columnList;
private String statisticsOutput = null;
@@ -76,6 +76,8 @@ public class FactDistinctColumnsReducer extends KylinReducer<SelfDefineSortableK
private int uhcReducerCount;
private Map<Integer, Integer> reducerIdToColumnIndex = new HashMap<>();
private int taskId;
+ private boolean isPartitionCol = false;
+ private int rowCount = 0;
//local build dict
private boolean isReducerLocalBuildDict;
@@ -84,7 +86,6 @@ public class FactDistinctColumnsReducer extends KylinReducer<SelfDefineSortableK
private long timeMinValue = Long.MAX_VALUE;
public static final String DICT_FILE_POSTFIX = ".rldict";
public static final String PARTITION_COL_INFO_FILE_POSTFIX = ".pci";
- private boolean isPartitionCol = false;
@Override
protected void setup(Context context) throws IOException {
@@ -111,6 +112,7 @@ public class FactDistinctColumnsReducer extends KylinReducer<SelfDefineSortableK
baseCuboidRowCountInMappers = Lists.newArrayList();
cuboidHLLMap = Maps.newHashMap();
samplingPercentage = Integer.parseInt(context.getConfiguration().get(BatchConstants.CFG_STATISTICS_SAMPLING_PERCENT));
+ logger.info("Reducer " + taskId + " handling stats");
} else if (collectStatistics && (taskId == numberOfTasks - 2)) {
// partition col
isStatistics = false;
@@ -120,6 +122,7 @@ public class FactDistinctColumnsReducer extends KylinReducer<SelfDefineSortableK
logger.info("Do not have partition col. This reducer will keep empty");
}
colValues = Lists.newLinkedList();
+ logger.info("Reducer " + taskId + " handling partition column " + col);
} else {
// normal col
isStatistics = false;
@@ -135,6 +138,7 @@ public class FactDistinctColumnsReducer extends KylinReducer<SelfDefineSortableK
builder = DictionaryGenerator.newDictionaryBuilder(col.getType());
builder.init(null, 0);
}
+ logger.info("Reducer " + taskId + " handling column " + col + ", isReducerLocalBuildDict=" + isReducerLocalBuildDict);
}
}
@@ -178,6 +182,7 @@ public class FactDistinctColumnsReducer extends KylinReducer<SelfDefineSortableK
} else if (isPartitionCol) {
// partition col
String value = Bytes.toString(key.getBytes(), 1, key.getLength() - 1);
+ logAFewRows(value);
long time = DateFormat.stringToMillis(value);
timeMinValue = Math.min(timeMinValue, time);
timeMaxValue = Math.max(timeMaxValue, time);
@@ -185,6 +190,7 @@ public class FactDistinctColumnsReducer extends KylinReducer<SelfDefineSortableK
// normal col
if (isReducerLocalBuildDict) {
String value = Bytes.toString(key.getBytes(), 1, key.getLength() - 1);
+ logAFewRows(value);
builder.addValue(value);
} else {
colValues.add(new ByteArray(Bytes.copy(key.getBytes(), 1, key.getLength() - 1)));
@@ -195,14 +201,22 @@ public class FactDistinctColumnsReducer extends KylinReducer<SelfDefineSortableK
}
}
}
+
+ rowCount++;
+ }
+
+ private void logAFewRows(String value) {
+ if (rowCount < 10) {
+ logger.info("Received value: " + value);
+ }
}
private void outputDistinctValues(TblColRef col, Collection<ByteArray> values, Context context) throws IOException {
final Configuration conf = context.getConfiguration();
final FileSystem fs = FileSystem.get(conf);
final String outputPath = conf.get(BatchConstants.CFG_OUTPUT_PATH);
- final Path colDir = new Path(outputPath, col.getName());
- final String fileName = col.getName() + "-" + taskId % uhcReducerCount;
+ final Path colDir = new Path(outputPath, col.getIdentity());
+ final String fileName = col.getIdentity() + "-" + taskId % uhcReducerCount;
final Path outputFile = new Path(colDir, fileName);
FSDataOutputStream out = null;
@@ -229,7 +243,7 @@ public class FactDistinctColumnsReducer extends KylinReducer<SelfDefineSortableK
}
private void outputDict(TblColRef col, Dictionary<String> dict, Context context) throws IOException {
- final String fileName = col.getName() + DICT_FILE_POSTFIX;
+ final String fileName = col.getIdentity() + DICT_FILE_POSTFIX;
FSDataOutputStream out = getOutputStream(context, fileName);
try {
String dictClassName = dict.getClass().getName();
@@ -242,7 +256,7 @@ public class FactDistinctColumnsReducer extends KylinReducer<SelfDefineSortableK
}
private void outputPartitionInfo(Context context) throws IOException {
- final String fileName = col.getName() + PARTITION_COL_INFO_FILE_POSTFIX;
+ final String fileName = col.getIdentity() + PARTITION_COL_INFO_FILE_POSTFIX;
FSDataOutputStream out = getOutputStream(context, fileName);
try {
out.writeLong(timeMinValue);
@@ -256,15 +270,12 @@ public class FactDistinctColumnsReducer extends KylinReducer<SelfDefineSortableK
private FSDataOutputStream getOutputStream(Context context, String outputFileName) throws IOException {
final Configuration conf = context.getConfiguration();
final FileSystem fs = FileSystem.get(conf);
- final String outputPath = conf.get(BatchConstants.CFG_OUTPUT_PATH);
- final Path colDir = new Path(outputPath, col.getName());
- final Path outputFile = new Path(colDir, outputFileName);
- FSDataOutputStream out = null;
- if (!fs.exists(colDir)) {
- fs.mkdirs(colDir);
+ final Path outputPath = new Path(conf.get(BatchConstants.CFG_OUTPUT_PATH));
+ final Path outputFile = new Path(outputPath, outputFileName);
+ if (!fs.exists(outputPath)) {
+ fs.mkdirs(outputPath);
}
- fs.deleteOnExit(outputFile);
- out = fs.create(outputFile);
+ FSDataOutputStream out = fs.create(outputFile);
return out;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/c6c67197/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctHiveColumnsMapper.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctHiveColumnsMapper.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctHiveColumnsMapper.java
index 5692c76..ed65343 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctHiveColumnsMapper.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctHiveColumnsMapper.java
@@ -31,6 +31,8 @@ import org.apache.kylin.engine.mr.common.BatchConstants;
import org.apache.kylin.measure.BufferedMeasureCodec;
import org.apache.kylin.measure.hllc.HLLCounter;
import org.apache.kylin.metadata.model.TblColRef;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import com.google.common.collect.Lists;
import com.google.common.hash.HashFunction;
@@ -41,6 +43,8 @@ import com.google.common.hash.Hashing;
*/
public class FactDistinctHiveColumnsMapper<KEYIN> extends FactDistinctColumnsMapperBase<KEYIN, Object> {
+ private static final Logger logger = LoggerFactory.getLogger(FactDistinctHiveColumnsMapper.class);
+
protected boolean collectStatistics = false;
protected CuboidScheduler cuboidScheduler = null;
protected int nRowKey;
@@ -51,7 +55,7 @@ public class FactDistinctHiveColumnsMapper<KEYIN> extends FactDistinctColumnsMap
private int rowCount = 0;
private int samplingPercentage;
private ByteArray[] row_hashcodes = null;
- private ByteBuffer keyBuffer;
+ private ByteBuffer tmpbuf;
private static final Text EMPTY_TEXT = new Text();
public static final byte MARK_FOR_PARTITION_COL = (byte) 0xFE;
public static final byte MARK_FOR_HLL = (byte) 0xFF;
@@ -62,7 +66,7 @@ public class FactDistinctHiveColumnsMapper<KEYIN> extends FactDistinctColumnsMap
@Override
protected void setup(Context context) throws IOException {
super.setup(context);
- keyBuffer = ByteBuffer.allocate(4096);
+ tmpbuf = ByteBuffer.allocate(4096);
collectStatistics = Boolean.parseBoolean(context.getConfiguration().get(BatchConstants.CFG_STATISTICS_ENABLED));
if (collectStatistics) {
samplingPercentage = Integer.parseInt(context.getConfiguration().get(BatchConstants.CFG_STATISTICS_SAMPLING_PERCENT));
@@ -127,55 +131,54 @@ public class FactDistinctHiveColumnsMapper<KEYIN> extends FactDistinctColumnsMap
public void doMap(KEYIN key, Object record, Context context) throws IOException, InterruptedException {
String[] row = flatTableInputFormat.parseMapperInput(record);
- keyBuffer.clear();
- try {
- for (int i = 0; i < factDictCols.size(); i++) {
- String fieldValue = row[dictionaryColumnIndex[i]];
- if (fieldValue == null)
- continue;
- int offset = keyBuffer.position();
-
- int reducerIndex;
- if (uhcIndex[i] == 0) {
- //for the normal dictionary column
- reducerIndex = columnIndexToReducerBeginId.get(i);
- } else {
- //for the uhc
- reducerIndex = columnIndexToReducerBeginId.get(i) + (fieldValue.hashCode() & 0x7fffffff) % uhcReducerCount;
- }
- keyBuffer.put(Bytes.toBytes(reducerIndex)[3]);
- keyBuffer.put(Bytes.toBytes(fieldValue));
- outputKey.set(keyBuffer.array(), offset, keyBuffer.position() - offset);
- sortableKey.setText(outputKey);
- //judge type
- sortableKey.setTypeIdByDatatype(factDictCols.get(i).getType());
- context.write(sortableKey, EMPTY_TEXT);
+ for (int i = 0; i < factDictCols.size(); i++) {
+ String fieldValue = row[dictionaryColumnIndex[i]];
+ if (fieldValue == null)
+ continue;
+
+ int reducerIndex;
+ if (uhcIndex[i] == 0) {
+ //for the normal dictionary column
+ reducerIndex = columnIndexToReducerBeginId.get(i);
+ } else {
+ //for the uhc
+ reducerIndex = columnIndexToReducerBeginId.get(i) + (fieldValue.hashCode() & 0x7fffffff) % uhcReducerCount;
+ }
+
+ tmpbuf.clear();
+ tmpbuf.put(Bytes.toBytes(reducerIndex)[3]);
+ tmpbuf.put(Bytes.toBytes(fieldValue));
+ outputKey.set(tmpbuf.array(), 0, tmpbuf.position());
+ sortableKey.setText(outputKey);
+ //judge type
+ sortableKey.setTypeIdByDatatype(factDictCols.get(i).getType());
+ context.write(sortableKey, EMPTY_TEXT);
+
+ // log a few rows for troubleshooting
+ if (rowCount < 10) {
+ logger.info("Sample output: " + factDictCols.get(i) + " '" + fieldValue + "' => reducer " + reducerIndex);
}
- } catch (Exception ex) {
- handleErrorRecord(row, ex);
}
if (collectStatistics) {
- if (rowCount < samplingPercentage) {
+ if (rowCount % 100 < samplingPercentage) {
putRowKeyToHLL(row);
}
if (needFetchPartitionCol == true) {
String fieldValue = row[partitionColumnIndex];
if (fieldValue != null) {
- int offset = keyBuffer.position();
- keyBuffer.put(MARK_FOR_PARTITION_COL);
- keyBuffer.put(Bytes.toBytes(fieldValue));
- outputKey.set(keyBuffer.array(), offset, keyBuffer.position() - offset);
+ tmpbuf.clear();
+ tmpbuf.put(MARK_FOR_PARTITION_COL);
+ tmpbuf.put(Bytes.toBytes(fieldValue));
+ outputKey.set(tmpbuf.array(), 0, tmpbuf.position());
sortableKey.setText(outputKey);
sortableKey.setTypeId((byte) 0);
context.write(sortableKey, EMPTY_TEXT);
}
}
}
-
- if (rowCount++ == 100)
- rowCount = 0;
+ rowCount++;
}
private void putRowKeyToHLL(String[] row) {
@@ -211,10 +214,10 @@ public class FactDistinctHiveColumnsMapper<KEYIN> extends FactDistinctColumnsMap
for (int i = 0; i < cuboidIds.length; i++) {
hll = allCuboidsHLL[i];
- keyBuffer.clear();
- keyBuffer.put(MARK_FOR_HLL); // one byte
- keyBuffer.putLong(cuboidIds[i]);
- outputKey.set(keyBuffer.array(), 0, keyBuffer.position());
+ tmpbuf.clear();
+ tmpbuf.put(MARK_FOR_HLL); // one byte
+ tmpbuf.putLong(cuboidIds[i]);
+ outputKey.set(tmpbuf.array(), 0, tmpbuf.position());
hllBuf.clear();
hll.writeRegisters(hllBuf);
outputValue.set(hllBuf.array(), 0, hllBuf.position());
http://git-wip-us.apache.org/repos/asf/kylin/blob/c6c67197/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java
index d3becfe..dcc9190 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java
@@ -59,7 +59,6 @@ public class UpdateCubeInfoAfterBuildStep extends AbstractExecutable {
long cubeSizeBytes = cubingJob.findCubeSizeBytes();
segment.setLastBuildJobID(CubingExecutableUtil.getCubingJobId(this.getParams()));
- segment.setIndexPath(CubingExecutableUtil.getIndexPath(this.getParams()));
segment.setLastBuildTime(System.currentTimeMillis());
segment.setSizeKB(cubeSizeBytes / 1024);
segment.setInputRecords(sourceCount);
@@ -81,8 +80,7 @@ public class UpdateCubeInfoAfterBuildStep extends AbstractExecutable {
private void updateTimeRange(CubeSegment segment) throws IOException {
final TblColRef partitionCol = segment.getCubeDesc().getModel().getPartitionDesc().getPartitionDateColumnRef();
final String factColumnsInputPath = this.getParams().get(BatchConstants.CFG_OUTPUT_PATH);
- Path colDir = new Path(factColumnsInputPath, partitionCol.getName());
- Path outputFile = new Path(colDir, partitionCol.getName() + FactDistinctColumnsReducer.PARTITION_COL_INFO_FILE_POSTFIX);
+ Path outputFile = new Path(factColumnsInputPath, partitionCol.getIdentity() + FactDistinctColumnsReducer.PARTITION_COL_INFO_FILE_POSTFIX);
FileSystem fs = HadoopUtil.getFileSystem(outputFile.toString());
FSDataInputStream is = null;
long minValue = Long.MAX_VALUE, maxValue = Long.MIN_VALUE;
http://git-wip-us.apache.org/repos/asf/kylin/blob/c6c67197/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterMergeStep.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterMergeStep.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterMergeStep.java
index d2fa73e..add5c42 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterMergeStep.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterMergeStep.java
@@ -75,7 +75,6 @@ public class UpdateCubeInfoAfterMergeStep extends AbstractExecutable {
mergedSegment.setInputRecords(sourceCount);
mergedSegment.setInputRecordsSize(sourceSize);
mergedSegment.setLastBuildJobID(CubingExecutableUtil.getCubingJobId(this.getParams()));
- mergedSegment.setIndexPath(CubingExecutableUtil.getIndexPath(this.getParams()));
mergedSegment.setLastBuildTime(System.currentTimeMillis());
try {
http://git-wip-us.apache.org/repos/asf/kylin/blob/c6c67197/examples/test_case_data/localmeta/cube_desc/ci_inner_join_cube.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube_desc/ci_inner_join_cube.json b/examples/test_case_data/localmeta/cube_desc/ci_inner_join_cube.json
index 287ead6..1ebd7f3 100644
--- a/examples/test_case_data/localmeta/cube_desc/ci_inner_join_cube.json
+++ b/examples/test_case_data/localmeta/cube_desc/ci_inner_join_cube.json
@@ -5,61 +5,96 @@
"description" : null,
"dimensions" : [ {
"name" : "CAL_DT",
- "table" : "EDW.TEST_CAL_DT",
+ "table" : "TEST_CAL_DT",
"column" : "{FK}",
"derived" : [ "WEEK_BEG_DT" ]
}, {
"name" : "ORDER_ID",
- "table" : "DEFAULT.TEST_KYLIN_FACT",
+ "table" : "TEST_KYLIN_FACT",
"column" : "ORDER_ID"
}, {
"name" : "TEST_DATE_ENC",
- "table" : "DEFAULT.TEST_ORDER",
+ "table" : "TEST_ORDER",
"column" : "TEST_DATE_ENC"
}, {
"name" : "TEST_TIME_ENC",
- "table" : "DEFAULT.TEST_ORDER",
+ "table" : "TEST_ORDER",
"column" : "TEST_TIME_ENC"
}, {
"name" : "CATEGORY",
- "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS",
+ "table" : "TEST_CATEGORY_GROUPINGS",
"column" : "{FK}",
"derived" : [ "USER_DEFINED_FIELD1", "USER_DEFINED_FIELD3", "UPD_DATE", "UPD_USER" ]
}, {
"name" : "CATEGORY_HIERARCHY",
- "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS",
+ "table" : "TEST_CATEGORY_GROUPINGS",
"column" : "META_CATEG_NAME",
"derived" : null
}, {
"name" : "CATEGORY_HIERARCHY",
- "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS",
+ "table" : "TEST_CATEGORY_GROUPINGS",
"column" : "CATEG_LVL2_NAME",
"derived" : null
}, {
"name" : "CATEGORY_HIERARCHY",
- "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS",
+ "table" : "TEST_CATEGORY_GROUPINGS",
"column" : "CATEG_LVL3_NAME",
"derived" : null
}, {
"name" : "LSTG_FORMAT_NAME",
- "table" : "DEFAULT.TEST_KYLIN_FACT",
+ "table" : "TEST_KYLIN_FACT",
"column" : "LSTG_FORMAT_NAME",
"derived" : null
}, {
"name" : "SITE_ID",
- "table" : "EDW.TEST_SITES",
+ "table" : "TEST_SITES",
"column" : "{FK}",
"derived" : [ "SITE_NAME", "CRE_USER" ]
}, {
"name" : "SELLER_TYPE_CD",
- "table" : "EDW.TEST_SELLER_TYPE_DIM",
+ "table" : "TEST_SELLER_TYPE_DIM",
"column" : "{FK}",
"derived" : [ "SELLER_TYPE_DESC" ]
}, {
"name" : "SELLER_ID",
- "table" : "DEFAULT.TEST_KYLIN_FACT",
- "column" : "SELLER_ID",
- "derived" : null
+ "table" : "TEST_KYLIN_FACT",
+ "column" : "SELLER_ID"
+ }, {
+ "name" : "SELLER_BUYER_LEVEL",
+ "table" : "SELLER_ACCOUNT",
+ "column" : "ACCOUNT_BUYER_LEVEL"
+ }, {
+ "name" : "SELLER_SELLER_LEVEL",
+ "table" : "SELLER_ACCOUNT",
+ "column" : "ACCOUNT_SELLER_LEVEL"
+ }, {
+ "name" : "SELLER_COUNTRY",
+ "table" : "SELLER_ACCOUNT",
+ "column" : "ACCOUNT_COUNTRY"
+ }, {
+ "name" : "SELLER_COUNTRY_NAME",
+ "table" : "SELLER_COUNTRY",
+ "column" : "NAME"
+ }, {
+ "name" : "BUYER_ID",
+ "table" : "TEST_ORDER",
+ "column" : "BUYER_ID"
+ }, {
+ "name" : "BUYER_BUYER_LEVEL",
+ "table" : "BUYER_ACCOUNT",
+ "column" : "ACCOUNT_BUYER_LEVEL"
+ }, {
+ "name" : "BUYER_SELLER_LEVEL",
+ "table" : "BUYER_ACCOUNT",
+ "column" : "ACCOUNT_SELLER_LEVEL"
+ }, {
+ "name" : "BUYER_COUNTRY",
+ "table" : "BUYER_ACCOUNT",
+ "column" : "ACCOUNT_COUNTRY"
+ }, {
+ "name" : "BUYER_COUNTRY_NAME",
+ "table" : "BUYER_COUNTRY",
+ "column" : "NAME"
} ],
"measures" : [ {
"name" : "TRANS_CNT",
@@ -77,7 +112,7 @@
"expression" : "SUM",
"parameter" : {
"type" : "column",
- "value" : "ITEM_COUNT"
+ "value" : "TEST_KYLIN_FACT.ITEM_COUNT"
},
"returntype" : "bigint"
}
@@ -87,7 +122,7 @@
"expression" : "SUM",
"parameter" : {
"type" : "column",
- "value" : "PRICE"
+ "value" : "TEST_KYLIN_FACT.PRICE"
},
"returntype" : "decimal(19,4)"
}
@@ -97,7 +132,7 @@
"expression" : "MIN",
"parameter" : {
"type" : "column",
- "value" : "PRICE"
+ "value" : "TEST_KYLIN_FACT.PRICE"
},
"returntype" : "decimal(19,4)"
}
@@ -107,7 +142,7 @@
"expression" : "MAX",
"parameter" : {
"type" : "column",
- "value" : "PRICE"
+ "value" : "TEST_KYLIN_FACT.PRICE"
},
"returntype" : "decimal(19,4)"
}
@@ -117,7 +152,7 @@
"expression" : "COUNT_DISTINCT",
"parameter" : {
"type" : "column",
- "value" : "SELLER_ID"
+ "value" : "TEST_KYLIN_FACT.SELLER_ID"
},
"returntype" : "hllc(10)"
}
@@ -127,10 +162,10 @@
"expression" : "COUNT_DISTINCT",
"parameter" : {
"type" : "column",
- "value" : "LSTG_FORMAT_NAME",
+ "value" : "TEST_KYLIN_FACT.LSTG_FORMAT_NAME",
"next_parameter" : {
"type" : "column",
- "value" : "SELLER_ID"
+ "value" : "TEST_KYLIN_FACT.SELLER_ID"
}
},
"returntype" : "hllc(10)"
@@ -141,10 +176,10 @@
"expression" : "TOP_N",
"parameter" : {
"type" : "column",
- "value" : "PRICE",
+ "value" : "TEST_KYLIN_FACT.PRICE",
"next_parameter" : {
"type" : "column",
- "value" : "SELLER_ID"
+ "value" : "TEST_KYLIN_FACT.SELLER_ID"
}
},
"returntype" : "topn(100)",
@@ -156,7 +191,7 @@
"expression" : "COUNT_DISTINCT",
"parameter" : {
"type" : "column",
- "value" : "TEST_COUNT_DISTINCT_BITMAP"
+ "value" : "TEST_KYLIN_FACT.TEST_COUNT_DISTINCT_BITMAP"
},
"returntype" : "bitmap"
}
@@ -166,10 +201,38 @@
"expression" : "EXTENDED_COLUMN",
"parameter": {
"type": "column",
- "value": "ORDER_ID",
+ "value": "TEST_KYLIN_FACT.ORDER_ID",
+ "next_parameter": {
+ "type": "column",
+ "value": "TEST_ORDER.TEST_EXTENDED_COLUMN"
+ }
+ },
+ "returntype": "extendedcolumn(100)"
+ }
+ }, {
+ "name" : "BUYER_CONTACT",
+ "function" : {
+ "expression" : "EXTENDED_COLUMN",
+ "parameter": {
+ "type": "column",
+ "value": "TEST_ORDER.BUYER_ID",
+ "next_parameter": {
+ "type": "column",
+ "value": "BUYER_ACCOUNT.ACCOUNT_CONTACT"
+ }
+ },
+ "returntype": "extendedcolumn(100)"
+ }
+ }, {
+ "name" : "SELLER_CONTACT",
+ "function" : {
+ "expression" : "EXTENDED_COLUMN",
+ "parameter": {
+ "type": "column",
+ "value": "TEST_KYLIN_FACT.SELLER_ID",
"next_parameter": {
"type": "column",
- "value": "TEST_EXTENDED_COLUMN"
+ "value": "SELLER_ACCOUNT.ACCOUNT_CONTACT"
}
},
"returntype": "extendedcolumn(100)"
@@ -180,7 +243,7 @@
"expression" : "RAW",
"parameter" : {
"type" : "column",
- "value" : "TRANS_ID"
+ "value" : "TEST_KYLIN_FACT.TRANS_ID"
},
"returntype" : "raw"
}
@@ -190,7 +253,7 @@
"expression" : "RAW",
"parameter" : {
"type" : "column",
- "value" : "PRICE"
+ "value" : "TEST_KYLIN_FACT.PRICE"
},
"returntype" : "raw"
}
@@ -200,52 +263,79 @@
"expression" : "RAW",
"parameter" : {
"type" : "column",
- "value" : "CAL_DT"
+ "value" : "TEST_KYLIN_FACT.CAL_DT"
},
"returntype" : "raw"
}
} ],
"dictionaries": [ {
- "column": "TEST_COUNT_DISTINCT_BITMAP",
+ "column": "TEST_KYLIN_FACT.TEST_COUNT_DISTINCT_BITMAP",
"builder": "org.apache.kylin.dict.GlobalDictionaryBuilder"
} ],
"rowkey" : {
"rowkey_columns" : [ {
- "column" : "SELLER_ID",
+ "column" : "TEST_KYLIN_FACT.SELLER_ID",
"encoding" : "int:4"
}, {
- "column" : "ORDER_ID",
+ "column" : "TEST_KYLIN_FACT.ORDER_ID",
"encoding" : "int:4"
}, {
- "column" : "CAL_DT",
+ "column" : "TEST_KYLIN_FACT.CAL_DT",
"encoding" : "dict"
}, {
- "column" : "LEAF_CATEG_ID",
+ "column" : "TEST_KYLIN_FACT.LEAF_CATEG_ID",
"encoding" : "dict"
}, {
- "column" : "META_CATEG_NAME",
+ "column" : "TEST_CATEGORY_GROUPINGS.META_CATEG_NAME",
"encoding" : "dict"
}, {
- "column" : "CATEG_LVL2_NAME",
+ "column" : "TEST_CATEGORY_GROUPINGS.CATEG_LVL2_NAME",
"encoding" : "dict"
}, {
- "column" : "CATEG_LVL3_NAME",
+ "column" : "TEST_CATEGORY_GROUPINGS.CATEG_LVL3_NAME",
"encoding" : "dict"
}, {
- "column" : "LSTG_FORMAT_NAME",
+ "column" : "TEST_KYLIN_FACT.LSTG_FORMAT_NAME",
"encoding" : "fixed_length:12"
}, {
- "column" : "LSTG_SITE_ID",
+ "column" : "TEST_KYLIN_FACT.LSTG_SITE_ID",
"encoding" : "dict"
}, {
- "column" : "SLR_SEGMENT_CD",
+ "column" : "TEST_KYLIN_FACT.SLR_SEGMENT_CD",
"encoding" : "dict"
}, {
- "column" : "TEST_TIME_ENC",
+ "column" : "TEST_ORDER.TEST_TIME_ENC",
"encoding" : "time"
}, {
- "column" : "TEST_DATE_ENC",
+ "column" : "TEST_ORDER.TEST_DATE_ENC",
"encoding" : "date"
+ }, {
+ "column" : "TEST_ORDER.BUYER_ID",
+ "encoding" : "int:4"
+ }, {
+ "column" : "BUYER_ACCOUNT.ACCOUNT_BUYER_LEVEL",
+ "encoding" : "dict"
+ }, {
+ "column" : "BUYER_ACCOUNT.ACCOUNT_SELLER_LEVEL",
+ "encoding" : "dict"
+ }, {
+ "column" : "BUYER_ACCOUNT.ACCOUNT_COUNTRY",
+ "encoding" : "dict"
+ }, {
+ "column" : "BUYER_COUNTRY.NAME",
+ "encoding" : "dict"
+ }, {
+ "column" : "SELLER_ACCOUNT.ACCOUNT_BUYER_LEVEL",
+ "encoding" : "dict"
+ }, {
+ "column" : "SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL",
+ "encoding" : "dict"
+ }, {
+ "column" : "SELLER_ACCOUNT.ACCOUNT_COUNTRY",
+ "encoding" : "dict"
+ }, {
+ "column" : "SELLER_COUNTRY.NAME",
+ "encoding" : "dict"
} ]
},
"signature" : null,
@@ -268,26 +358,80 @@
"name" : "f3",
"columns" : [ {
"qualifier" : "m",
- "measure_refs" : [ "TEST_EXTENDED_COLUMN", "TRANS_ID_RAW", "PRICE_RAW", "CAL_DT_RAW" ]
+ "measure_refs" : [ "TEST_EXTENDED_COLUMN", "TRANS_ID_RAW", "PRICE_RAW", "CAL_DT_RAW", "BUYER_CONTACT", "SELLER_CONTACT" ]
} ]
} ]
},
"aggregation_groups" : [ {
- "includes" : [ "CAL_DT", "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME", "LEAF_CATEG_ID",
- "LSTG_FORMAT_NAME", "LSTG_SITE_ID", "SLR_SEGMENT_CD" ],
+ "includes" : [ "TEST_KYLIN_FACT.CAL_DT",
+ "TEST_KYLIN_FACT.LEAF_CATEG_ID",
+ "TEST_KYLIN_FACT.LSTG_FORMAT_NAME",
+ "TEST_KYLIN_FACT.LSTG_SITE_ID",
+ "TEST_KYLIN_FACT.SLR_SEGMENT_CD",
+ "TEST_CATEGORY_GROUPINGS.META_CATEG_NAME",
+ "TEST_CATEGORY_GROUPINGS.CATEG_LVL2_NAME",
+ "TEST_CATEGORY_GROUPINGS.CATEG_LVL3_NAME" ],
"select_rule" : {
- "hierarchy_dims" : [ [ "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME", "LEAF_CATEG_ID" ] ],
+ "hierarchy_dims" : [ [ "TEST_CATEGORY_GROUPINGS.META_CATEG_NAME",
+ "TEST_CATEGORY_GROUPINGS.CATEG_LVL2_NAME",
+ "TEST_CATEGORY_GROUPINGS.CATEG_LVL3_NAME",
+ "TEST_KYLIN_FACT.LEAF_CATEG_ID" ] ],
"mandatory_dims" : [ ],
- "joint_dims" : [ [ "LSTG_FORMAT_NAME", "LSTG_SITE_ID", "SLR_SEGMENT_CD" ] ]
+ "joint_dims" : [ [ "TEST_KYLIN_FACT.LSTG_FORMAT_NAME",
+ "TEST_KYLIN_FACT.LSTG_SITE_ID",
+ "TEST_KYLIN_FACT.SLR_SEGMENT_CD" ] ]
}
}, {
- "includes" : [ "CAL_DT", "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME", "LEAF_CATEG_ID", "LSTG_FORMAT_NAME",
- "LSTG_SITE_ID", "SLR_SEGMENT_CD", "SELLER_ID",
- "ORDER_ID", "TEST_DATE_ENC", "TEST_TIME_ENC" ],
+ "includes" : [ "TEST_KYLIN_FACT.CAL_DT",
+ "TEST_KYLIN_FACT.LEAF_CATEG_ID",
+ "TEST_KYLIN_FACT.LSTG_FORMAT_NAME",
+ "TEST_KYLIN_FACT.LSTG_SITE_ID",
+ "TEST_KYLIN_FACT.SLR_SEGMENT_CD",
+ "TEST_CATEGORY_GROUPINGS.META_CATEG_NAME",
+ "TEST_CATEGORY_GROUPINGS.CATEG_LVL2_NAME",
+ "TEST_CATEGORY_GROUPINGS.CATEG_LVL3_NAME",
+
+ "TEST_KYLIN_FACT.SELLER_ID",
+ "SELLER_ACCOUNT.ACCOUNT_BUYER_LEVEL",
+ "SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL",
+ "SELLER_ACCOUNT.ACCOUNT_COUNTRY",
+ "SELLER_COUNTRY.NAME",
+
+ "TEST_KYLIN_FACT.ORDER_ID",
+ "TEST_ORDER.TEST_DATE_ENC",
+ "TEST_ORDER.TEST_TIME_ENC",
+ "TEST_ORDER.BUYER_ID",
+ "BUYER_ACCOUNT.ACCOUNT_BUYER_LEVEL",
+ "BUYER_ACCOUNT.ACCOUNT_SELLER_LEVEL",
+ "BUYER_ACCOUNT.ACCOUNT_COUNTRY",
+ "BUYER_COUNTRY.NAME" ],
"select_rule" : {
"hierarchy_dims" : [ ],
- "mandatory_dims" : [ "CAL_DT", "SELLER_ID", "ORDER_ID", "TEST_DATE_ENC", "TEST_TIME_ENC" ],
- "joint_dims" : [ [ "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME", "LEAF_CATEG_ID" ], [ "LSTG_FORMAT_NAME", "LSTG_SITE_ID", "SLR_SEGMENT_CD" ] ]
+ "mandatory_dims" : [ "TEST_KYLIN_FACT.CAL_DT" ],
+
+ "joint_dims" : [ [ "TEST_CATEGORY_GROUPINGS.META_CATEG_NAME",
+ "TEST_CATEGORY_GROUPINGS.CATEG_LVL2_NAME",
+ "TEST_CATEGORY_GROUPINGS.CATEG_LVL3_NAME",
+ "TEST_KYLIN_FACT.LEAF_CATEG_ID" ],
+
+ [ "TEST_KYLIN_FACT.LSTG_FORMAT_NAME",
+ "TEST_KYLIN_FACT.LSTG_SITE_ID",
+ "TEST_KYLIN_FACT.SLR_SEGMENT_CD" ],
+
+ [ "TEST_KYLIN_FACT.SELLER_ID",
+ "SELLER_ACCOUNT.ACCOUNT_BUYER_LEVEL",
+ "SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL",
+ "SELLER_ACCOUNT.ACCOUNT_COUNTRY",
+ "SELLER_COUNTRY.NAME" ],
+
+ [ "TEST_KYLIN_FACT.ORDER_ID",
+ "TEST_ORDER.TEST_DATE_ENC",
+ "TEST_ORDER.TEST_TIME_ENC",
+ "TEST_ORDER.BUYER_ID",
+ "BUYER_ACCOUNT.ACCOUNT_BUYER_LEVEL",
+ "BUYER_ACCOUNT.ACCOUNT_SELLER_LEVEL",
+ "BUYER_ACCOUNT.ACCOUNT_COUNTRY",
+ "BUYER_COUNTRY.NAME" ] ]
}
} ],
"notify_list" : null,
http://git-wip-us.apache.org/repos/asf/kylin/blob/c6c67197/examples/test_case_data/localmeta/cube_desc/ci_left_join_cube.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube_desc/ci_left_join_cube.json b/examples/test_case_data/localmeta/cube_desc/ci_left_join_cube.json
index be1ea99..4048b6e 100644
--- a/examples/test_case_data/localmeta/cube_desc/ci_left_join_cube.json
+++ b/examples/test_case_data/localmeta/cube_desc/ci_left_join_cube.json
@@ -5,61 +5,96 @@
"description" : null,
"dimensions" : [ {
"name" : "CAL_DT",
- "table" : "EDW.TEST_CAL_DT",
+ "table" : "TEST_CAL_DT",
"column" : "{FK}",
"derived" : [ "WEEK_BEG_DT" ]
}, {
"name" : "ORDER_ID",
- "table" : "DEFAULT.TEST_KYLIN_FACT",
+ "table" : "TEST_KYLIN_FACT",
"column" : "ORDER_ID"
}, {
"name" : "TEST_DATE_ENC",
- "table" : "DEFAULT.TEST_ORDER",
+ "table" : "TEST_ORDER",
"column" : "TEST_DATE_ENC"
}, {
"name" : "TEST_TIME_ENC",
- "table" : "DEFAULT.TEST_ORDER",
+ "table" : "TEST_ORDER",
"column" : "TEST_TIME_ENC"
}, {
"name" : "CATEGORY",
- "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS",
+ "table" : "TEST_CATEGORY_GROUPINGS",
"column" : "{FK}",
"derived" : [ "USER_DEFINED_FIELD1", "USER_DEFINED_FIELD3", "UPD_DATE", "UPD_USER" ]
}, {
"name" : "CATEGORY_HIERARCHY",
- "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS",
+ "table" : "TEST_CATEGORY_GROUPINGS",
"column" : "META_CATEG_NAME",
"derived" : null
}, {
"name" : "CATEGORY_HIERARCHY",
- "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS",
+ "table" : "TEST_CATEGORY_GROUPINGS",
"column" : "CATEG_LVL2_NAME",
"derived" : null
}, {
"name" : "CATEGORY_HIERARCHY",
- "table" : "DEFAULT.TEST_CATEGORY_GROUPINGS",
+ "table" : "TEST_CATEGORY_GROUPINGS",
"column" : "CATEG_LVL3_NAME",
"derived" : null
}, {
"name" : "LSTG_FORMAT_NAME",
- "table" : "DEFAULT.TEST_KYLIN_FACT",
+ "table" : "TEST_KYLIN_FACT",
"column" : "LSTG_FORMAT_NAME",
"derived" : null
}, {
"name" : "SITE_ID",
- "table" : "EDW.TEST_SITES",
+ "table" : "TEST_SITES",
"column" : "{FK}",
"derived" : [ "SITE_NAME", "CRE_USER" ]
}, {
"name" : "SELLER_TYPE_CD",
- "table" : "EDW.TEST_SELLER_TYPE_DIM",
+ "table" : "TEST_SELLER_TYPE_DIM",
"column" : "{FK}",
"derived" : [ "SELLER_TYPE_DESC" ]
}, {
"name" : "SELLER_ID",
- "table" : "DEFAULT.TEST_KYLIN_FACT",
- "column" : "SELLER_ID",
- "derived" : null
+ "table" : "TEST_KYLIN_FACT",
+ "column" : "SELLER_ID"
+ }, {
+ "name" : "SELLER_BUYER_LEVEL",
+ "table" : "SELLER_ACCOUNT",
+ "column" : "ACCOUNT_BUYER_LEVEL"
+ }, {
+ "name" : "SELLER_SELLER_LEVEL",
+ "table" : "SELLER_ACCOUNT",
+ "column" : "ACCOUNT_SELLER_LEVEL"
+ }, {
+ "name" : "SELLER_COUNTRY",
+ "table" : "SELLER_ACCOUNT",
+ "column" : "ACCOUNT_COUNTRY"
+ }, {
+ "name" : "SELLER_COUNTRY_NAME",
+ "table" : "SELLER_COUNTRY",
+ "column" : "NAME"
+ }, {
+ "name" : "BUYER_ID",
+ "table" : "TEST_ORDER",
+ "column" : "BUYER_ID"
+ }, {
+ "name" : "BUYER_BUYER_LEVEL",
+ "table" : "BUYER_ACCOUNT",
+ "column" : "ACCOUNT_BUYER_LEVEL"
+ }, {
+ "name" : "BUYER_SELLER_LEVEL",
+ "table" : "BUYER_ACCOUNT",
+ "column" : "ACCOUNT_SELLER_LEVEL"
+ }, {
+ "name" : "BUYER_COUNTRY",
+ "table" : "BUYER_ACCOUNT",
+ "column" : "ACCOUNT_COUNTRY"
+ }, {
+ "name" : "BUYER_COUNTRY_NAME",
+ "table" : "BUYER_COUNTRY",
+ "column" : "NAME"
} ],
"measures" : [ {
"name" : "TRANS_CNT",
@@ -77,7 +112,7 @@
"expression" : "SUM",
"parameter" : {
"type" : "column",
- "value" : "ITEM_COUNT"
+ "value" : "TEST_KYLIN_FACT.ITEM_COUNT"
},
"returntype" : "bigint"
}
@@ -87,7 +122,7 @@
"expression" : "SUM",
"parameter" : {
"type" : "column",
- "value" : "PRICE"
+ "value" : "TEST_KYLIN_FACT.PRICE"
},
"returntype" : "decimal(19,4)"
}
@@ -97,7 +132,7 @@
"expression" : "MIN",
"parameter" : {
"type" : "column",
- "value" : "PRICE"
+ "value" : "TEST_KYLIN_FACT.PRICE"
},
"returntype" : "decimal(19,4)"
}
@@ -107,7 +142,7 @@
"expression" : "MAX",
"parameter" : {
"type" : "column",
- "value" : "PRICE"
+ "value" : "TEST_KYLIN_FACT.PRICE"
},
"returntype" : "decimal(19,4)"
}
@@ -117,7 +152,7 @@
"expression" : "COUNT_DISTINCT",
"parameter" : {
"type" : "column",
- "value" : "SELLER_ID"
+ "value" : "TEST_KYLIN_FACT.SELLER_ID"
},
"returntype" : "hllc(10)"
}
@@ -127,10 +162,10 @@
"expression" : "COUNT_DISTINCT",
"parameter" : {
"type" : "column",
- "value" : "LSTG_FORMAT_NAME",
+ "value" : "TEST_KYLIN_FACT.LSTG_FORMAT_NAME",
"next_parameter" : {
"type" : "column",
- "value" : "SELLER_ID"
+ "value" : "TEST_KYLIN_FACT.SELLER_ID"
}
},
"returntype" : "hllc(10)"
@@ -141,10 +176,10 @@
"expression" : "TOP_N",
"parameter" : {
"type" : "column",
- "value" : "PRICE",
+ "value" : "TEST_KYLIN_FACT.PRICE",
"next_parameter" : {
"type" : "column",
- "value" : "SELLER_ID"
+ "value" : "TEST_KYLIN_FACT.SELLER_ID"
}
},
"returntype" : "topn(100)",
@@ -156,7 +191,7 @@
"expression" : "COUNT_DISTINCT",
"parameter" : {
"type" : "column",
- "value" : "TEST_COUNT_DISTINCT_BITMAP"
+ "value" : "TEST_KYLIN_FACT.TEST_COUNT_DISTINCT_BITMAP"
},
"returntype" : "bitmap"
}
@@ -166,10 +201,38 @@
"expression" : "EXTENDED_COLUMN",
"parameter": {
"type": "column",
- "value": "ORDER_ID",
+ "value": "TEST_KYLIN_FACT.ORDER_ID",
+ "next_parameter": {
+ "type": "column",
+ "value": "TEST_ORDER.TEST_EXTENDED_COLUMN"
+ }
+ },
+ "returntype": "extendedcolumn(100)"
+ }
+ }, {
+ "name" : "BUYER_CONTACT",
+ "function" : {
+ "expression" : "EXTENDED_COLUMN",
+ "parameter": {
+ "type": "column",
+ "value": "TEST_ORDER.BUYER_ID",
+ "next_parameter": {
+ "type": "column",
+ "value": "BUYER_ACCOUNT.ACCOUNT_CONTACT"
+ }
+ },
+ "returntype": "extendedcolumn(100)"
+ }
+ }, {
+ "name" : "SELLER_CONTACT",
+ "function" : {
+ "expression" : "EXTENDED_COLUMN",
+ "parameter": {
+ "type": "column",
+ "value": "TEST_KYLIN_FACT.SELLER_ID",
"next_parameter": {
"type": "column",
- "value": "TEST_EXTENDED_COLUMN"
+ "value": "SELLER_ACCOUNT.ACCOUNT_CONTACT"
}
},
"returntype": "extendedcolumn(100)"
@@ -180,7 +243,7 @@
"expression" : "RAW",
"parameter" : {
"type" : "column",
- "value" : "TRANS_ID"
+ "value" : "TEST_KYLIN_FACT.TRANS_ID"
},
"returntype" : "raw"
}
@@ -190,7 +253,7 @@
"expression" : "RAW",
"parameter" : {
"type" : "column",
- "value" : "PRICE"
+ "value" : "TEST_KYLIN_FACT.PRICE"
},
"returntype" : "raw"
}
@@ -200,52 +263,79 @@
"expression" : "RAW",
"parameter" : {
"type" : "column",
- "value" : "CAL_DT"
+ "value" : "TEST_KYLIN_FACT.CAL_DT"
},
"returntype" : "raw"
}
} ],
"dictionaries": [ {
- "column": "TEST_COUNT_DISTINCT_BITMAP",
+ "column": "TEST_KYLIN_FACT.TEST_COUNT_DISTINCT_BITMAP",
"builder": "org.apache.kylin.dict.GlobalDictionaryBuilder"
} ],
"rowkey" : {
"rowkey_columns" : [ {
- "column" : "SELLER_ID",
+ "column" : "TEST_KYLIN_FACT.SELLER_ID",
"encoding" : "int:4"
}, {
- "column" : "ORDER_ID",
+ "column" : "TEST_KYLIN_FACT.ORDER_ID",
"encoding" : "int:4"
}, {
- "column" : "CAL_DT",
+ "column" : "TEST_KYLIN_FACT.CAL_DT",
"encoding" : "dict"
}, {
- "column" : "LEAF_CATEG_ID",
+ "column" : "TEST_KYLIN_FACT.LEAF_CATEG_ID",
"encoding" : "dict"
}, {
- "column" : "META_CATEG_NAME",
+ "column" : "TEST_CATEGORY_GROUPINGS.META_CATEG_NAME",
"encoding" : "dict"
}, {
- "column" : "CATEG_LVL2_NAME",
+ "column" : "TEST_CATEGORY_GROUPINGS.CATEG_LVL2_NAME",
"encoding" : "dict"
}, {
- "column" : "CATEG_LVL3_NAME",
+ "column" : "TEST_CATEGORY_GROUPINGS.CATEG_LVL3_NAME",
"encoding" : "dict"
}, {
- "column" : "LSTG_FORMAT_NAME",
+ "column" : "TEST_KYLIN_FACT.LSTG_FORMAT_NAME",
"encoding" : "fixed_length:12"
}, {
- "column" : "LSTG_SITE_ID",
+ "column" : "TEST_KYLIN_FACT.LSTG_SITE_ID",
"encoding" : "dict"
}, {
- "column" : "SLR_SEGMENT_CD",
+ "column" : "TEST_KYLIN_FACT.SLR_SEGMENT_CD",
"encoding" : "dict"
}, {
- "column" : "TEST_TIME_ENC",
+ "column" : "TEST_ORDER.TEST_TIME_ENC",
"encoding" : "time"
}, {
- "column" : "TEST_DATE_ENC",
+ "column" : "TEST_ORDER.TEST_DATE_ENC",
"encoding" : "date"
+ }, {
+ "column" : "TEST_ORDER.BUYER_ID",
+ "encoding" : "int:4"
+ }, {
+ "column" : "BUYER_ACCOUNT.ACCOUNT_BUYER_LEVEL",
+ "encoding" : "dict"
+ }, {
+ "column" : "BUYER_ACCOUNT.ACCOUNT_SELLER_LEVEL",
+ "encoding" : "dict"
+ }, {
+ "column" : "BUYER_ACCOUNT.ACCOUNT_COUNTRY",
+ "encoding" : "dict"
+ }, {
+ "column" : "BUYER_COUNTRY.NAME",
+ "encoding" : "dict"
+ }, {
+ "column" : "SELLER_ACCOUNT.ACCOUNT_BUYER_LEVEL",
+ "encoding" : "dict"
+ }, {
+ "column" : "SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL",
+ "encoding" : "dict"
+ }, {
+ "column" : "SELLER_ACCOUNT.ACCOUNT_COUNTRY",
+ "encoding" : "dict"
+ }, {
+ "column" : "SELLER_COUNTRY.NAME",
+ "encoding" : "dict"
} ]
},
"signature" : null,
@@ -268,26 +358,80 @@
"name" : "f3",
"columns" : [ {
"qualifier" : "m",
- "measure_refs" : [ "TEST_EXTENDED_COLUMN", "TRANS_ID_RAW", "PRICE_RAW", "CAL_DT_RAW" ]
+ "measure_refs" : [ "TEST_EXTENDED_COLUMN", "TRANS_ID_RAW", "PRICE_RAW", "CAL_DT_RAW", "BUYER_CONTACT", "SELLER_CONTACT" ]
} ]
} ]
},
"aggregation_groups" : [ {
- "includes" : [ "CAL_DT", "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME", "LEAF_CATEG_ID",
- "LSTG_FORMAT_NAME", "LSTG_SITE_ID", "SLR_SEGMENT_CD" ],
+ "includes" : [ "TEST_KYLIN_FACT.CAL_DT",
+ "TEST_KYLIN_FACT.LEAF_CATEG_ID",
+ "TEST_KYLIN_FACT.LSTG_FORMAT_NAME",
+ "TEST_KYLIN_FACT.LSTG_SITE_ID",
+ "TEST_KYLIN_FACT.SLR_SEGMENT_CD",
+ "TEST_CATEGORY_GROUPINGS.META_CATEG_NAME",
+ "TEST_CATEGORY_GROUPINGS.CATEG_LVL2_NAME",
+ "TEST_CATEGORY_GROUPINGS.CATEG_LVL3_NAME" ],
"select_rule" : {
- "hierarchy_dims" : [ [ "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME", "LEAF_CATEG_ID" ] ],
+ "hierarchy_dims" : [ [ "TEST_CATEGORY_GROUPINGS.META_CATEG_NAME",
+ "TEST_CATEGORY_GROUPINGS.CATEG_LVL2_NAME",
+ "TEST_CATEGORY_GROUPINGS.CATEG_LVL3_NAME",
+ "TEST_KYLIN_FACT.LEAF_CATEG_ID" ] ],
"mandatory_dims" : [ ],
- "joint_dims" : [ [ "LSTG_FORMAT_NAME", "LSTG_SITE_ID", "SLR_SEGMENT_CD" ] ]
+ "joint_dims" : [ [ "TEST_KYLIN_FACT.LSTG_FORMAT_NAME",
+ "TEST_KYLIN_FACT.LSTG_SITE_ID",
+ "TEST_KYLIN_FACT.SLR_SEGMENT_CD" ] ]
}
}, {
- "includes" : [ "CAL_DT", "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME", "LEAF_CATEG_ID", "LSTG_FORMAT_NAME",
- "LSTG_SITE_ID", "SLR_SEGMENT_CD", "SELLER_ID",
- "ORDER_ID", "TEST_DATE_ENC", "TEST_TIME_ENC" ],
+ "includes" : [ "TEST_KYLIN_FACT.CAL_DT",
+ "TEST_KYLIN_FACT.LEAF_CATEG_ID",
+ "TEST_KYLIN_FACT.LSTG_FORMAT_NAME",
+ "TEST_KYLIN_FACT.LSTG_SITE_ID",
+ "TEST_KYLIN_FACT.SLR_SEGMENT_CD",
+ "TEST_CATEGORY_GROUPINGS.META_CATEG_NAME",
+ "TEST_CATEGORY_GROUPINGS.CATEG_LVL2_NAME",
+ "TEST_CATEGORY_GROUPINGS.CATEG_LVL3_NAME",
+
+ "TEST_KYLIN_FACT.SELLER_ID",
+ "SELLER_ACCOUNT.ACCOUNT_BUYER_LEVEL",
+ "SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL",
+ "SELLER_ACCOUNT.ACCOUNT_COUNTRY",
+ "SELLER_COUNTRY.NAME",
+
+ "TEST_KYLIN_FACT.ORDER_ID",
+ "TEST_ORDER.TEST_DATE_ENC",
+ "TEST_ORDER.TEST_TIME_ENC",
+ "TEST_ORDER.BUYER_ID",
+ "BUYER_ACCOUNT.ACCOUNT_BUYER_LEVEL",
+ "BUYER_ACCOUNT.ACCOUNT_SELLER_LEVEL",
+ "BUYER_ACCOUNT.ACCOUNT_COUNTRY",
+ "BUYER_COUNTRY.NAME" ],
"select_rule" : {
"hierarchy_dims" : [ ],
- "mandatory_dims" : [ "CAL_DT", "SELLER_ID", "ORDER_ID", "TEST_DATE_ENC", "TEST_TIME_ENC" ],
- "joint_dims" : [ [ "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME", "LEAF_CATEG_ID" ], [ "LSTG_FORMAT_NAME", "LSTG_SITE_ID", "SLR_SEGMENT_CD" ] ]
+ "mandatory_dims" : [ "TEST_KYLIN_FACT.CAL_DT" ],
+
+ "joint_dims" : [ [ "TEST_CATEGORY_GROUPINGS.META_CATEG_NAME",
+ "TEST_CATEGORY_GROUPINGS.CATEG_LVL2_NAME",
+ "TEST_CATEGORY_GROUPINGS.CATEG_LVL3_NAME",
+ "TEST_KYLIN_FACT.LEAF_CATEG_ID" ],
+
+ [ "TEST_KYLIN_FACT.LSTG_FORMAT_NAME",
+ "TEST_KYLIN_FACT.LSTG_SITE_ID",
+ "TEST_KYLIN_FACT.SLR_SEGMENT_CD" ],
+
+ [ "TEST_KYLIN_FACT.SELLER_ID",
+ "SELLER_ACCOUNT.ACCOUNT_BUYER_LEVEL",
+ "SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL",
+ "SELLER_ACCOUNT.ACCOUNT_COUNTRY",
+ "SELLER_COUNTRY.NAME" ],
+
+ [ "TEST_KYLIN_FACT.ORDER_ID",
+ "TEST_ORDER.TEST_DATE_ENC",
+ "TEST_ORDER.TEST_TIME_ENC",
+ "TEST_ORDER.BUYER_ID",
+ "BUYER_ACCOUNT.ACCOUNT_BUYER_LEVEL",
+ "BUYER_ACCOUNT.ACCOUNT_SELLER_LEVEL",
+ "BUYER_ACCOUNT.ACCOUNT_COUNTRY",
+ "BUYER_COUNTRY.NAME" ] ]
}
} ],
"notify_list" : null,
http://git-wip-us.apache.org/repos/asf/kylin/blob/c6c67197/examples/test_case_data/localmeta/cube_desc/test_kylin_snowflake_sales_cube.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube_desc/test_kylin_snowflake_sales_cube.json b/examples/test_case_data/localmeta/cube_desc/test_kylin_snowflake_sales_cube.json
deleted file mode 100644
index ee0d68c..0000000
--- a/examples/test_case_data/localmeta/cube_desc/test_kylin_snowflake_sales_cube.json
+++ /dev/null
@@ -1,268 +0,0 @@
-{
- "uuid" : "0ef9b7a8-3929-4dff-b59d-2100aadc8dbf",
- "last_modified" : 1451468470824,
- "name" : "test_kylin_snowflake_sales_cube",
- "model_name" : "test_kylin_snowflake_sales_model",
- "description" : null,
- "null_string" : null,
- "dimensions" : [ {
- "name" : "TRANS_ID",
- "table" : "SNOWTEST.KYLIN_SALES",
- "column" : "TRANS_ID",
- "derived" : null
- }, {
- "name" : "CAL_DT",
- "table" : "SNOWTEST.KYLIN_CAL_DT",
- "column" : "{FK}",
- "derived" : [ "WEEK_BEG_DT", "MONTH_BEG_DT", "YEAR_BEG_DT" ]
- }, {
- "name" : "CATEGORY",
- "table" : "SNOWTEST.KYLIN_CATEGORY_GROUPINGS",
- "column" : "{FK}",
- "derived" : [ "USER_DEFINED_FIELD1", "USER_DEFINED_FIELD3" ]
- }, {
- "name" : "META_CATEG_NAME",
- "table" : "SNOWTEST.KYLIN_CATEGORY_GROUPINGS",
- "column" : "META_CATEG_NAME",
- "derived" : null
- }, {
- "name" : "CATEG_LVL2_NAME",
- "table" : "SNOWTEST.KYLIN_CATEGORY_GROUPINGS",
- "column" : "CATEG_LVL2_NAME",
- "derived" : null
- }, {
- "name" : "CATEG_LVL3_NAME",
- "table" : "SNOWTEST.KYLIN_CATEGORY_GROUPINGS",
- "column" : "CATEG_LVL3_NAME",
- "derived" : null
- }, {
- "name" : "LSTG_FORMAT_NAME",
- "table" : "SNOWTEST.KYLIN_SALES",
- "column" : "LSTG_FORMAT_NAME",
- "derived" : null
- }, {
- "name" : "SELLER_ID",
- "table" : "SNOWTEST.KYLIN_SALES",
- "column" : "SELLER_ID",
- "derived" : null
- }, {
- "name" : "BUYER_ID",
- "table" : "SNOWTEST.KYLIN_SALES",
- "column" : "BUYER_ID",
- "derived" : null
- }, {
- "name" : "ACCOUNT_BUYER_LEVEL",
- "table" : "BUYER_ACCOUNT",
- "column" : "ACCOUNT_BUYER_LEVEL",
- "derived" : null
- }, {
- "name" : "ACCOUNT_SELLER_LEVEL",
- "table" : "SELLER_ACCOUNT",
- "column" : "ACCOUNT_SELLER_LEVEL",
- "derived" : null
- }, {
- "name" : "BUYER_COUNTRY",
- "table" : "BUYER_ACCOUNT",
- "column" : "ACCOUNT_COUNTRY",
- "derived" : null
- }, {
- "name" : "SELLER_COUNTRY",
- "table" : "SELLER_ACCOUNT",
- "column" : "ACCOUNT_COUNTRY",
- "derived" : null
- }, {
- "name" : "BUYER_COUNTRY_NAME",
- "table" : "BUYER_COUNTRY",
- "column" : "NAME",
- "derived" : null
- }, {
- "name" : "SELLER_COUNTRY_NAME",
- "table" : "SELLER_COUNTRY",
- "column" : "NAME",
- "derived" : null
- }, {
- "name" : "OPS_USER_ID",
- "table" : "SNOWTEST.KYLIN_SALES",
- "column" : "OPS_USER_ID",
- "derived" : null
- }, {
- "name" : "OPS_REGION",
- "table" : "SNOWTEST.KYLIN_SALES",
- "column" : "OPS_REGION",
- "derived" : null
- } ],
- "measures" : [ {
- "name" : "GMV_SUM",
- "function" : {
- "expression" : "SUM",
- "parameter" : {
- "type" : "column",
- "value" : "PRICE",
- "next_parameter" : null
- },
- "returntype" : "decimal(19,4)"
- }
- }, {
- "name" : "GMV_MIN",
- "function" : {
- "expression" : "MIN",
- "parameter" : {
- "type" : "column",
- "value" : "PRICE",
- "next_parameter" : null
- },
- "returntype" : "decimal(19,4)"
- }
- }, {
- "name" : "GMV_MAX",
- "function" : {
- "expression" : "MAX",
- "parameter" : {
- "type" : "column",
- "value" : "PRICE",
- "next_parameter" : null
- },
- "returntype" : "decimal(19,4)"
- }
- }, {
- "name" : "TRANS_CNT",
- "function" : {
- "expression" : "COUNT",
- "parameter" : {
- "type" : "constant",
- "value" : "1",
- "next_parameter" : null
- },
- "returntype" : "bigint"
- }
- }, {
- "name" : "SELLER_CNT_HLL",
- "function" : {
- "expression" : "COUNT_DISTINCT",
- "parameter" : {
- "type" : "column",
- "value" : "SELLER_ID",
- "next_parameter" : null
- },
- "returntype" : "hllc(10)"
- }
- }, {
- "name" : "TOP_SELLER",
- "function" : {
- "expression" : "TOP_N",
- "parameter" : {
- "type" : "column",
- "value" : "PRICE",
- "next_parameter" : {
- "type" : "column",
- "value" : "SELLER_ID",
- "next_parameter" : null
- }
- },
- "returntype" : "topn(100)"
- }
- } ],
- "rowkey" : {
- "rowkey_columns" : [ {
- "column" : "BUYER_ID",
- "encoding" : "integer:4"
- }, {
- "column" : "SELLER_ID",
- "encoding" : "integer:4"
- }, {
- "column" : "TRANS_ID",
- "encoding" : "integer:4"
- }, {
- "column" : "PART_DT",
- "encoding" : "date"
- }, {
- "column" : "LEAF_CATEG_ID",
- "encoding" : "dict"
- }, {
- "column" : "META_CATEG_NAME",
- "encoding" : "dict"
- }, {
- "column" : "CATEG_LVL2_NAME",
- "encoding" : "dict"
- }, {
- "column" : "CATEG_LVL3_NAME",
- "encoding" : "dict"
- }, {
- "column" : "BUYER_ACCOUNT.ACCOUNT_BUYER_LEVEL",
- "encoding" : "dict"
- }, {
- "column" : "SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL",
- "encoding" : "dict"
- }, {
- "column" : "BUYER_ACCOUNT.ACCOUNT_COUNTRY",
- "encoding" : "dict"
- }, {
- "column" : "SELLER_ACCOUNT.ACCOUNT_COUNTRY",
- "encoding" : "dict"
- }, {
- "column" : "BUYER_COUNTRY.NAME",
- "encoding" : "dict"
- }, {
- "column" : "SELLER_COUNTRY.NAME",
- "encoding" : "dict"
- }, {
- "column" : "LSTG_FORMAT_NAME",
- "encoding" : "dict"
- }, {
- "column" : "LSTG_SITE_ID",
- "encoding" : "dict"
- }, {
- "column" : "OPS_USER_ID",
- "encoding" : "dict"
- }, {
- "column" : "OPS_REGION",
- "encoding" : "dict"
- } ]
- },
- "hbase_mapping" : {
- "column_family" : [ {
- "name" : "F1",
- "columns" : [ {
- "qualifier" : "M",
- "measure_refs" : [ "GMV_SUM", "GMV_MIN", "GMV_MAX", "TRANS_CNT" ]
- } ]
- }, {
- "name" : "F2",
- "columns" : [ {
- "qualifier" : "M",
- "measure_refs" : [ "SELLER_CNT_HLL", "TOP_SELLER" ]
- } ]
- } ]
- },
- "aggregation_groups" : [ {
- "includes" : [ "PART_DT", "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME", "LEAF_CATEG_ID", "LSTG_FORMAT_NAME", "LSTG_SITE_ID", "OPS_USER_ID", "OPS_REGION",
- "BUYER_ACCOUNT.ACCOUNT_BUYER_LEVEL", "SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL", "BUYER_ACCOUNT.ACCOUNT_COUNTRY", "SELLER_ACCOUNT.ACCOUNT_COUNTRY", "BUYER_COUNTRY.NAME", "SELLER_COUNTRY.NAME" ],
- "select_rule" : {
- "hierarchy_dims" : [ [ "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME", "LEAF_CATEG_ID" ] ],
- "mandatory_dims" : [ "PART_DT" ],
- "joint_dims" : [ [ "BUYER_ACCOUNT.ACCOUNT_COUNTRY", "BUYER_COUNTRY.NAME" ], [ "SELLER_ACCOUNT.ACCOUNT_COUNTRY", "SELLER_COUNTRY.NAME" ],
- [ "BUYER_ACCOUNT.ACCOUNT_BUYER_LEVEL", "SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL" ], [ "LSTG_FORMAT_NAME", "LSTG_SITE_ID" ], [ "OPS_USER_ID", "OPS_REGION" ] ]
- }
- }, {
- "includes" : [ "TRANS_ID", "BUYER_ID", "SELLER_ID",
- "PART_DT", "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME", "LEAF_CATEG_ID", "LSTG_FORMAT_NAME", "LSTG_SITE_ID", "OPS_USER_ID", "OPS_REGION",
- "BUYER_ACCOUNT.ACCOUNT_BUYER_LEVEL", "SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL", "BUYER_ACCOUNT.ACCOUNT_COUNTRY", "SELLER_ACCOUNT.ACCOUNT_COUNTRY", "BUYER_COUNTRY.NAME", "SELLER_COUNTRY.NAME" ],
- "select_rule" : {
- "hierarchy_dims" : [ ],
- "mandatory_dims" : [ "TRANS_ID", "BUYER_ID", "SELLER_ID",
- "PART_DT", "META_CATEG_NAME", "CATEG_LVL2_NAME", "CATEG_LVL3_NAME", "LEAF_CATEG_ID", "LSTG_FORMAT_NAME", "LSTG_SITE_ID", "OPS_USER_ID", "OPS_REGION",
- "BUYER_ACCOUNT.ACCOUNT_BUYER_LEVEL", "SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL", "BUYER_ACCOUNT.ACCOUNT_COUNTRY", "SELLER_ACCOUNT.ACCOUNT_COUNTRY", "BUYER_COUNTRY.NAME", "SELLER_COUNTRY.NAME" ],
- "joint_dims" : [ ]
- }
- } ],
- "notify_list" : null,
- "status_need_notify" : [ ],
- "partition_date_start" : 1325376000000,
- "auto_merge_time_ranges" : null,
- "retention_range" : 0,
- "engine_type" : 2,
- "storage_type" : 2,
- "override_kylin_properties" : {
- "kylin.cube.aggrgroup.is-mandatory-only-valid" : "true"
- }
-}
[4/7] kylin git commit: KYLIN-2320 fix getSizeOfId() for empty
TrieDictionaryForest
Posted by li...@apache.org.
KYLIN-2320 fix getSizeOfId() for empty TrieDictionaryForest
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/30d3763f
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/30d3763f
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/30d3763f
Branch: refs/heads/KYLIN-2314
Commit: 30d3763f8256a9e88ae9620499021e99771b3351
Parents: 73a4029
Author: Yang Li <li...@apache.org>
Authored: Sat Dec 24 07:34:12 2016 +0800
Committer: Yang Li <li...@apache.org>
Committed: Sat Dec 24 14:46:38 2016 +0800
----------------------------------------------------------------------
.../apache/kylin/dict/TrieDictionaryForest.java | 4 +-
.../kylin/dict/TrieDictionaryForestTest.java | 41 ++++++++++++++++----
2 files changed, 35 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/30d3763f/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryForest.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryForest.java b/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryForest.java
index 1015b11..e746348 100755
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryForest.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryForest.java
@@ -87,7 +87,7 @@ public class TrieDictionaryForest<T> extends Dictionary<T> {
@Override
public int getSizeOfId() {
if (trees.isEmpty())
- return -1;
+ return 1;
int maxOffset = accuOffset.get(accuOffset.size() - 1);
TrieDictionary<T> lastTree = trees.get(trees.size() - 1);
int sizeOfId = BytesUtil.sizeForValue(baseId + maxOffset + lastTree.getMaxId() + 1);
@@ -96,7 +96,7 @@ public class TrieDictionaryForest<T> extends Dictionary<T> {
@Override
public int getSizeOfValue() {
- int maxValue = -1;
+ int maxValue = 0;
for (TrieDictionary<T> tree : trees)
maxValue = Math.max(maxValue, tree.getSizeOfValue());
return maxValue;
http://git-wip-us.apache.org/repos/asf/kylin/blob/30d3763f/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryForestTest.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryForestTest.java b/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryForestTest.java
index 32a6ab5..238bb31 100755
--- a/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryForestTest.java
+++ b/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryForestTest.java
@@ -52,6 +52,14 @@ import org.junit.Test;
public class TrieDictionaryForestTest {
@Test
+ public void testEmptyDict() {
+ ArrayList<String> strs = new ArrayList<String>();
+ TrieDictionaryForestBuilder<String> builder = newDictBuilder(strs, 0);
+ TrieDictionaryForest<String> dict = builder.build();
+ assertSameBehaviorAsTrie(dict, strs, 0);
+ }
+
+ @Test
public void testBasicFound() {
ArrayList<String> strs = new ArrayList<String>();
strs.add("part");
@@ -67,11 +75,11 @@ public class TrieDictionaryForestTest {
dict.dump(System.out);
int expectId = baseId;
for (String s : strs) {
- System.out.println("value:" + s + " expect id:" + expectId);
assertEquals(expectId, dict.getIdFromValue(s));
expectId++;
}
- System.out.println("test ok");
+
+ assertSameBehaviorAsTrie(dict, strs, baseId);
}
@Test //one string one tree
@@ -95,11 +103,11 @@ public class TrieDictionaryForestTest {
assertEquals(strs.size(), dict.getTrees().size());
int expectId = baseId;
for (String s : strs) {
- System.out.println("value:" + s + " expect id:" + expectId);
assertEquals(expectId, dict.getIdFromValue(s));
expectId++;
}
- System.out.println("test ok");
+
+ assertSameBehaviorAsTrie(dict, strs, baseId);
}
@Test
@@ -115,11 +123,13 @@ public class TrieDictionaryForestTest {
dict.dump(System.out);
//null value query
int id = dict.getIdFromValue(null, 0);
- System.out.println(id);
+ assertEquals(255, id);
id = dict.getIdFromValue(null, 1);
- System.out.println(id);
+ assertEquals(255, id);
id = dict.getIdFromValue(null, -1);
- System.out.println(id);
+ assertEquals(255, id);
+
+ assertSameBehaviorAsTrie(dict, strs, 0);
}
@Test
@@ -940,7 +950,7 @@ public class TrieDictionaryForestTest {
b.addValue(strs.next());
return b;
}
-
+
private static class RandomStrings implements Iterable<String> {
final private int size;
@@ -1022,4 +1032,19 @@ public class TrieDictionaryForestTest {
return testData;
}
+ private void assertSameBehaviorAsTrie(TrieDictionaryForest<String> dict, ArrayList<String> strs, int baseId) {
+ TrieDictionaryBuilder<String> trieBuilder = new TrieDictionaryBuilder<>(new StringBytesConverter());
+ for (String s : strs) {
+ if (s != null)
+ trieBuilder.addValue(s);
+ }
+ TrieDictionary<String> trie = trieBuilder.build(baseId);
+
+ assertEquals(trie.getMaxId(), dict.getMaxId());
+ assertEquals(trie.getMinId(), dict.getMinId());
+ assertEquals(trie.getSize(), dict.getSize());
+ assertEquals(trie.getSizeOfId(), dict.getSizeOfId());
+ assertEquals(trie.getSizeOfValue(), dict.getSizeOfValue());
+ }
+
}
[2/7] kylin git commit: KYLIN-2287 rm deprecated code & add streaming
info when show storage
Posted by li...@apache.org.
KYLIN-2287 rm deprecated code & add streaming info when show storage
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/488283d3
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/488283d3
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/488283d3
Branch: refs/heads/KYLIN-2314
Commit: 488283d3de1a385a4a6ed03ebd2e412cb9f2f7d7
Parents: 4496928
Author: zhongjian <ji...@163.com>
Authored: Fri Dec 23 21:35:45 2016 +0800
Committer: zhongjian <ji...@163.com>
Committed: Fri Dec 23 21:35:45 2016 +0800
----------------------------------------------------------------------
webapp/app/js/controllers/cube.js | 6 ++++++
webapp/app/js/controllers/cubeEdit.js | 3 ---
webapp/app/js/controllers/sourceMeta.js | 3 ---
webapp/app/partials/cubes/cube_detail.html | 2 +-
4 files changed, 7 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/488283d3/webapp/app/js/controllers/cube.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/cube.js b/webapp/app/js/controllers/cube.js
index 57b53e7..9e675d2 100755
--- a/webapp/app/js/controllers/cube.js
+++ b/webapp/app/js/controllers/cube.js
@@ -88,6 +88,12 @@ KylinApp.controller('CubeCtrl', function ($scope, AccessService, MessageService,
CubeService.getHbaseInfo({cubeId: cube.name, propValue: null, action: null}, function (hbase) {
cube.hbase = hbase;
+ TableService.get({tableName:cube.model.fact_table},function(table) {
+ if (table && table.source_type == 1) {
+ cube.streaming = true;
+ }
+ })
+
// Calculate cube total size based on each htable.
var totalSize = 0;
hbase.forEach(function(t) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/488283d3/webapp/app/js/controllers/cubeEdit.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/cubeEdit.js b/webapp/app/js/controllers/cubeEdit.js
index 90cdbc0..edbb421 100755
--- a/webapp/app/js/controllers/cubeEdit.js
+++ b/webapp/app/js/controllers/cubeEdit.js
@@ -311,9 +311,6 @@ KylinApp.controller('CubeEditCtrl', function ($scope, $q, $routeParams, $locatio
"mode": 'edit'
};
- $scope.cubeState={
- "isStreaming": false
- }
//fetch cube info and model info in edit model
// ~ init
http://git-wip-us.apache.org/repos/asf/kylin/blob/488283d3/webapp/app/js/controllers/sourceMeta.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/sourceMeta.js b/webapp/app/js/controllers/sourceMeta.js
index dc3c421..bbb9915 100755
--- a/webapp/app/js/controllers/sourceMeta.js
+++ b/webapp/app/js/controllers/sourceMeta.js
@@ -552,9 +552,6 @@ KylinApp
var StreamingSourceCtrl = function ($scope, $location,$interpolate,$templateCache, $modalInstance, tableNames, MessageService, projectName, scope, tableConfig,cubeConfig,StreamingModel,StreamingService) {
- $scope.cubeState={
- "isStreaming": false
- }
$scope.state={
'mode':'edit'
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/488283d3/webapp/app/partials/cubes/cube_detail.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/cubes/cube_detail.html b/webapp/app/partials/cubes/cube_detail.html
index 1896b42..4319b94 100755
--- a/webapp/app/partials/cubes/cube_detail.html
+++ b/webapp/app/partials/cubes/cube_detail.html
@@ -44,7 +44,7 @@
<div class="cube-detail" ng-if="!cube.visiblePage || cube.visiblePage=='metadata'">
<div ng-include="'partials/cubes/cube_schema.html'" ng-controller="CubeSchemaCtrl"
- ng-init="state={mode:'view', cubeName:cube.name};cubeState={isStreaming:(!!cube.streaming)?true:false}"></div>
+ ng-init="state={mode:'view', cubeName:cube.name};"></div>
</div>
<div ng-show="cube.visiblePage=='sql'" class="cube-detail">