You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2024/09/23 11:15:33 UTC
(kylin) branch doc5.0 updated: Refine the tutorial of Model
This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch doc5.0
in repository https://gitbox.apache.org/repos/asf/kylin.git
The following commit(s) were added to refs/heads/doc5.0 by this push:
new e25c10db5e Refine the tutorial of Model
e25c10db5e is described below
commit e25c10db5ee3ca1b073743d2002c4d91419a64e5
Author: Pengfei Zhan <de...@gmail.com>
AuthorDate: Sun Sep 15 21:13:20 2024 +0800
Refine the tutorial of Model
---
website/docs/datasource/data_sampling.md | 4 +-
website/docs/datasource/import_hive.md | 4 +-
website/docs/icons/add_icon.svg | 5 +
website/docs/icons/batch_add_icon.svg | 4 +
website/docs/icons/batch_remove_icon.svg | 9 +
website/docs/icons/build_icon.svg | 5 +
website/docs/icons/circle_add_icon2.svg | 6 +
website/docs/icons/date_search.svg | 5 +
website/docs/icons/info_icon.svg | 7 +
website/docs/icons/look_details_icon.svg | 5 +
website/docs/icons/rect_add_icon.svg | 4 +
website/docs/icons/remove_icon.svg | 10 +
website/docs/icons/settings_icon.svg | 7 +
website/docs/icons/wizard_icon.svg | 8 +
website/docs/model/features/excluded_column.md | 22 ++
.../features}/fast_bitmap.md | 22 +-
.../features/images/fast_bitmap}/add_index.png | Bin
.../features/images/fast_bitmap}/model.png | Bin
.../images/fast_bitmap}/model_config_1.png | Bin
.../images/fast_bitmap}/model_config_2.png | Bin
.../features/images/fast_bitmap}/query_new.png | Bin
.../features/images/fast_bitmap}/query_old.png | Bin
.../images/fast_bitmap}/spark_plan_new.png | Bin
.../images/fast_bitmap}/spark_plan_old.png | Bin
.../measures/collect_set}/add_collect_set.png | Bin
.../measures/collect_set}/collect_result.png | Bin
.../measures/collect_set}/concatws_result.png | Bin
.../images/measures/corr}/corr_add_measure.png | Bin
.../images/measures/corr}/corr_edit_measure.png | Bin
.../features/images/measures/corr}/corr_query.png | Bin
.../cd_measures_add_precisely.png | Bin
.../count_distinct_hllc}/cd_measures_edit.png | Bin
.../percentile}/percentile_approximate.png | Bin
.../measures/percentile}/percentile_result.png | Bin
.../percentile}/percentile_result_hive.png | Bin
.../images/measures/topn}/topN_measure_edit.png | Bin
.../features/images/measures/topn}/topn_result.png | Bin
.../multilevel_partioning_build_subp_first.png | Bin
.../multilevel_partioning_build_subp_second.png | Bin
.../multilevel_partioning_build_subp_second_0.png | Bin
.../multilevel_partioning_build_subp_second_1.png | Bin
.../multilevel_partitioning_close.png | Bin
.../multilevel_partitioning_model_save.png | Bin
.../multilevel_partitioning_query.png | Bin
.../multilevel_partitioning_set.png | Bin
.../multilevel_partitioning_subp_value.png | Bin
.../scd2/historical_dimension_table_switch.png | Bin
.../features/images/scd2}/model_SCD1.png | Bin
.../features/images/scd2}/model_SCD2.png | Bin
.../features}/images/scd2/model_SCD2_5x.png | Bin
.../scd2/model_historical_dimension_table_scd2.png | Bin
.../model_historical_dimension_table_scd2_join.png | Bin
.../features}/integer_encoding.md | 6 +-
.../features/measures}/collect_set.md | 10 +-
.../features/measures}/corr.md | 6 +-
.../features/measures}/count_distinct_bitmap.md | 2 +-
.../features/measures}/count_distinct_hllc.md | 2 +-
.../features/measures}/intro.md | 0
.../features/measures}/percentile_approx.md | 10 +-
.../features/measures}/topn.md | 4 +-
.../features/multi_partitioning.md} | 26 +-
.../features/runtime_join.md} | 6 +-
.../features}/scalar_subquery.md | 8 +-
.../features/scd2.md} | 7 +-
website/docs/model/images/volume_per_time.png | Bin 0 -> 30104 bytes
website/docs/model/intro.md | 68 ++++
website/docs/model/manage/configuration.md | 17 +
website/docs/model/manage/data_loading.md | 144 +++++++
.../images/data_loading}/by_date/buildcube_0.png | Bin
.../images/data_loading}/by_date/buildcube_1.png | Bin
.../images/data_loading}/by_date/buildcube_2.png | Bin
.../images/data_loading}/by_date/buildcube_3.png | Bin
.../images/data_loading}/full_load/full_load.png | Bin
.../images/data_loading}/full_load/load_data.png | Bin
.../images/data_loading}/full_load/notice.png | Bin
.../data_loading}/load_by_date/add_segment.png | Bin
.../data_loading}/load_by_date/build_all_index.png | Bin
.../load_by_date/build_all_index2.png | Bin
.../data_loading}/load_by_date/build_index.png | Bin
.../load_by_date/build_index_by_segment.png | Bin
.../data_loading}/load_by_date/delete_index.png | Bin
.../data_loading}/load_by_date/load_data.png | Bin
.../images/data_loading}/load_by_date/load_ok.png | Bin
.../data_loading}/load_by_date/load_twice.png | Bin
.../data_loading}/load_by_date/model_list_3.png | Bin
.../images/data_loading}/load_by_date/notice_2.png | Bin
.../manage/images}/dimensions.png | Bin
.../model_overview => model/manage/images}/er.png | Bin
.../manage/images/import_export}/model_check.png | Bin
.../manage/images/import_export}/model_export.png | Bin
.../manage/images/import_export}/model_upload.png | Bin
.../manage/images}/measures.png | Bin
.../manage}/images/model_list.png | Bin
.../manage}/images/modellist_more_info.png | Bin
.../manage/images/publish}/model_publish.png | Bin
.../manage/images/segment}/segment.png | Bin
.../manage/images}/unfold_model.png | Bin
.../manage/import_export.md} | 16 +-
website/docs/model/manage/model_info.md | 87 +++++
website/docs/model/manage/model_operation.md | 53 +++
.../segment_merge.md => model/manage/segment.md} | 88 ++++-
website/docs/model/manual/agg_group_example.md | 156 ++++++++
website/docs/model/manual/aggregation_group.md | 255 +++++++++++++
website/docs/model/manual/computed_column.md | 187 ++++++++++
.../docs/model/manual/images/agg/add_agg_group.png | Bin 0 -> 258579 bytes
.../model/manual/images/agg/advanced_settings.gif | Bin 0 -> 1899824 bytes
.../manual}/images/agg/agg_detail.png | Bin
.../model/manual/images/agg/agg_group_page.png | Bin 0 -> 232927 bytes
.../model/manual/images/agg/defined_agg_group.png | Bin 0 -> 280272 bytes
.../manual/images/agg/examples}/AGG-4.png | Bin
.../manual/images/agg/examples}/AGG-6.png | Bin
.../manual/images/agg/examples}/Hierarchy-4.png | Bin
.../manual/images/agg/examples}/Mandatory-4.png | Bin
.../model/manual/images/agg/include_dimensions.png | Bin 0 -> 292161 bytes
.../model/manual/images/agg/include_measures.png | Bin 0 -> 209343 bytes
.../docs/model/manual/images/agg/index_view.png | Bin 0 -> 257308 bytes
.../manual/images/agg/principles/agg_group_p1.png} | Bin
.../manual/images/agg/principles/agg_group_p2.png} | Bin
.../manual/images/agg/principles/hierarchy_p1.png} | Bin
.../manual/images/agg/principles/hierarchy_p2.png} | Bin
.../manual/images/agg/principles/joint_p1.png} | Bin
.../manual/images/agg/principles/mandatory_p1.png} | Bin
.../manual/images/agg/principles/mandatory_p2.png} | Bin
.../manual/images/agg/principles/mdc_p1.png} | Bin
website/docs/model/manual/images/agg/recognize.gif | Bin 0 -> 1000358 bytes
.../model/manual/images/agg/to_agg_group_page.gif | Bin 0 -> 314865 bytes
.../manual/images/computed_column/add_cc_icon.svg | 4 +
.../manual/images/computed_column/add_cc_page.png | Bin 0 -> 238164 bytes
.../images/computed_column/add_dim_of_cc.png | Bin 0 -> 182755 bytes
.../images/computed_column/add_measure_of_cc.png | Bin 0 -> 280363 bytes
.../manual/images/computed_column/cc_editing.gif | Bin 0 -> 982654 bytes
.../manual/images/computed_column/cc_for_index.gif | Bin 0 -> 3614708 bytes
.../manual/images/computed_column/cc_on_table.png | Bin 0 -> 218358 bytes
.../manual/images/computed_column/nested_cc.png | Bin 0 -> 250242 bytes
.../manual/images/computed_column/where_is_cc.png | Bin 0 -> 217592 bytes
.../model/manual/images/modeling/add_dimension.gif | Bin 0 -> 3021685 bytes
.../manual/images/modeling/add_join_relations.png | Bin 0 -> 223190 bytes
.../model/manual/images/modeling/add_measure.gif | Bin 0 -> 6152699 bytes
.../manual/images/modeling}/add_model_set.png | Bin
.../model/manual/images/modeling/added_tables.png | Bin 0 -> 225644 bytes
.../images/modeling/create_join_relations.gif | Bin 0 -> 901093 bytes
.../manual/images/modeling/data_loading_ways.png | Bin 0 -> 223692 bytes
.../model/manual/images/modeling/date_format.svg | 11 +
.../manual/images/modeling/finished_modeling.png | Bin 0 -> 143888 bytes
.../manual/images/modeling/incremental_loading.png | Bin 0 -> 243177 bytes
.../model/manual/images/modeling/joined_tables.png | Bin 0 -> 223707 bytes
.../model/manual/images/modeling/model_list.png | Bin 0 -> 117677 bytes
.../model/manual/images/modeling/modeling1.png | Bin 0 -> 135141 bytes
.../images/modeling/redirect_to_add_index.png | Bin 0 -> 242228 bytes
.../manual/images/modeling/set_fact_table.png | Bin 0 -> 205206 bytes
.../manual/images/table_index/add_table_index.png | Bin 0 -> 280740 bytes
.../manual/images/table_index/index_overview.png | Bin 0 -> 259770 bytes
.../manual/images/table_index/table_index_list.png | Bin 0 -> 278588 bytes
.../images/table_index/to_tableindex_page.gif | Bin 0 -> 443001 bytes
website/docs/model/manual/modeling.md | 234 ++++++++++++
website/docs/model/manual/table_index.md | 59 +++
.../rec}/basic_concept_actions.md | 4 +-
.../rec}/images/flexible_index.png | Bin
.../rec}/images/low_usage.png | Bin
.../rec/images/qh_recommend/adjust_rec_conf.png | Bin 0 -> 240219 bytes
.../model/rec/images/qh_recommend/rec_rules.png | Bin 0 -> 261173 bytes
.../rec/images/qh_recommend/recommendations.png | Bin 0 -> 211033 bytes
.../rec}/images/recommendation_detail_en.png | Bin
.../rec}/images/recommendation_preferences_en.png | Bin
.../rec}/images/recommendations.en.png | Bin
.../rec/images/sql_modeling/convert_success.png | Bin 0 -> 226977 bytes
.../rec/images/sql_modeling/convert_to_index.png | Bin 0 -> 159192 bytes
.../rec/images/sql_modeling/create_new_model.png | Bin 0 -> 169216 bytes
.../rec/images/sql_modeling/create_or_convert.png | Bin 0 -> 199829 bytes
.../model/rec/images/sql_modeling/from_sql.png | Bin 0 -> 182680 bytes
.../model/rec/images/sql_modeling/imported_sql.png | Bin 0 -> 203094 bytes
.../rec/images/sql_modeling/left_join_model.png | Bin 0 -> 188974 bytes
.../model/rec/images/sql_modeling/model_list.png | Bin 0 -> 228728 bytes
.../rec/images/sql_modeling/preview_models.png | Bin 0 -> 220877 bytes
.../images/sql_modeling/show_accepted_index.png | Bin 0 -> 288060 bytes
.../model/rec/images/sql_modeling/upload_sql.png | Bin 0 -> 149879 bytes
.../model/rec/images/turn_on_recommendation.gif | Bin 0 -> 5486475 bytes
.../rec}/index_optimization.md | 4 +-
website/docs/model/rec/intro.md | 32 ++
website/docs/model/rec/optimize_by_qh.md | 136 +++++++
.../rec}/optimize_index/intro.md | 0
.../rec}/optimize_index/review_index.md | 15 +-
website/docs/model/rec/rule_setting.md | 74 ++++
website/docs/model/rec/sql_modeling.md | 138 +++++++
.../snapshot/images}/snapshot_1.png | Bin
.../snapshot/images}/snapshot_2.png | Bin
.../snapshot/images}/snapshot_3.png | Bin
.../snapshot/images}/snapshot_4.png | Bin
.../snapshot/images}/snapshot_5.png | Bin
.../snapshot/images}/snapshot_6.png | Bin
.../snapshot/images}/snapshot_7.png | Bin
.../snapshot/images}/snapshot_8.png | Bin
.../snapshot/images}/snapshot_9.png | Bin
.../snapshot/snapshot.md} | 24 +-
.../real-time => model/streaming}/custom-parser.md | 0
.../real-time => model/streaming}/images/add.png | Bin
.../images/custom_project_configuration_en.png | Bin
.../streaming}/images/index_data_range_en.png | Bin
.../streaming}/images/load_tables_1_en.png | Bin
.../streaming}/images/load_tables_2_en.png | Bin
.../real-time => model/streaming}/images/more.png | Bin
.../streaming}/images/par_config.png | Bin
.../streaming}/images/real-time_add_hive_table.png | Bin
.../images/real-time_add_kafka_table.png | Bin
.../streaming}/images/real-time_add_model.png | Bin
.../streaming}/images/real-time_add_model_pic.png | Bin
.../streaming}/images/real-time_arch.png | Bin
.../streaming}/images/real-time_arch1.png | Bin
.../images/real-time_associate_with_hive.png | Bin
.../images/real-time_build_batch_index.png | Bin
.../streaming}/images/real-time_build_index.png | Bin
.../images/real-time_build_index_time_range.png | Bin
.../images/real-time_build_realtime_index.png | Bin
.../real-time_dimension_table_timed_refresh.png | Bin
.../images/real-time_finish_add_source.png | Bin
.../images/real-time_kafka_message_sample.png | Bin
.../images/real-time_monitor_batch_status.png | Bin
.../images/real-time_monitor_realtime_config.png | Bin
.../real-time_monitor_realtime_diagnostics.png | Bin
.../images/real-time_monitor_realtime_logs.png | Bin
.../real-time_monitor_realtime_logs_info.png | Bin
.../images/real-time_monitor_realtime_status.png | Bin
.../images/real-time_query_batch_data.png | Bin
.../images/real-time_query_streaming_data.png | Bin
.../streaming}/images/real-time_topic.png | Bin
.../streaming}/images/select_custom_parser.png | Bin
.../real-time => model/streaming}/intro.md | 0
.../real-time => model/streaming}/prerequisite.md | 0
.../real-time => model/streaming}/real-time.md | 0
.../images/streaming_sdk_create_project.en.png | Bin
.../images/streaming_sdk_create_table_csv_1.en.png | Bin
.../images/streaming_sdk_create_table_csv_2.en.png | Bin
.../images/streaming_sdk_create_table_csv_3.en.png | Bin
.../streaming_sdk_create_table_json_1.en.png | Bin
.../streaming_sdk_create_table_json_2.en.png | Bin
.../streaming_sdk_create_table_json_3.en.png | Bin
.../streaming_sdk/images/streaming_sdk_lib.en.png | Bin
.../images/streaming_sdk_metainf.en.png | Bin
.../images/streaming_sdk_package.en.png | Bin
.../streaming}/streaming_sdk/streaming_sdk.md | 0
.../modeling/auto_modeling/data_modeling_by_SQL.md | 90 -----
.../auto_modeling/images/acceleration_rule.en.png | Bin 233612 -> 0 bytes
.../auto_modeling/images/ai_augment_principle.png | Bin 64616 -> 0 bytes
.../docs/modeling/auto_modeling/images/build.png | Bin 718 -> 0 bytes
.../auto_modeling/images/build_index_en.png | Bin 45256 -> 0 bytes
.../modeling/auto_modeling/images/formulas.en.png | Bin 31322 -> 0 bytes
.../docs/modeling/auto_modeling/images/fromsql.png | Bin 56817 -> 0 bytes
.../auto_modeling/images/import_sql_en.png | Bin 148611 -> 0 bytes
.../modeling/auto_modeling/images/main_flow.en.png | Bin 94907 -> 0 bytes
.../modeling/auto_modeling/images/preview_en.png | Bin 70635 -> 0 bytes
.../auto_modeling/images/rule_setting.en.png | Bin 153741 -> 0 bytes
.../modeling/auto_modeling/images/turn_on_rec.png | Bin 66768 -> 0 bytes
.../docs/modeling/auto_modeling/images/view.png | Bin 577 -> 0 bytes
.../docs/modeling/auto_modeling/images/wizard.png | Bin 449 -> 0 bytes
website/docs/modeling/auto_modeling/intro.md | 29 --
.../docs/modeling/auto_modeling/rule_setting.md | 99 -----
website/docs/modeling/images/add_dimention.gif | Bin 279896 -> 0 bytes
.../docs/modeling/images/add_join_relations.png | Bin 29579 -> 0 bytes
website/docs/modeling/images/add_measure.gif | Bin 189064 -> 0 bytes
website/docs/modeling/images/add_tables.png | Bin 45712 -> 0 bytes
.../docs/modeling/images/create_join_relations.gif | Bin 279896 -> 0 bytes
website/docs/modeling/images/reduceio.png | Bin 155351 -> 0 bytes
.../modeling/images/responsetime_datavolume.png | Bin 94683 -> 0 bytes
website/docs/modeling/images/save_load_method.png | Bin 54968 -> 0 bytes
.../docs/modeling/images/star_model_created.png | Bin 51718 -> 0 bytes
.../docs/modeling/images/switch_to_fact_table.png | Bin 33622 -> 0 bytes
website/docs/modeling/images/time_format.png | Bin 655 -> 0 bytes
website/docs/modeling/intro.md | 63 ----
website/docs/modeling/load_data/build_index.md | 47 ---
website/docs/modeling/load_data/by_date.md | 68 ----
website/docs/modeling/load_data/full_build.md | 41 --
.../load_data/images/build_empty_index_en.png | Bin 42914 -> 0 bytes
.../modeling/load_data/images/build_summary.png | Bin 141411 -> 0 bytes
.../images/customize_build_save_model.png | Bin 72711 -> 0 bytes
.../customize_build_save_model_partition.png | Bin 69557 -> 0 bytes
.../docs/modeling/load_data/images/full_build.png | Bin 32928 -> 0 bytes
.../modeling/load_data/images/full_build_cube.png | Bin 36363 -> 0 bytes
.../load_data/images/full_build_segment_info.png | Bin 37628 -> 0 bytes
.../docs/modeling/load_data/images/segment_2.png | Bin 146180 -> 0 bytes
.../docs/modeling/load_data/images/streaming/g.png | Bin 132606 -> 0 bytes
.../docs/modeling/load_data/images/streaming/h.png | Bin 84180 -> 0 bytes
.../docs/modeling/load_data/images/streaming/i.png | Bin 69850 -> 0 bytes
.../docs/modeling/load_data/images/streaming/j.png | Bin 115502 -> 0 bytes
.../docs/modeling/load_data/images/streaming/k.png | Bin 121139 -> 0 bytes
.../docs/modeling/load_data/images/streaming/l.png | Bin 138717 -> 0 bytes
.../docs/modeling/load_data/images/streaming/m.png | Bin 202509 -> 0 bytes
.../docs/modeling/load_data/images/streaming/n.png | Bin 117677 -> 0 bytes
.../docs/modeling/load_data/images/time_axis.png | Bin 31394 -> 0 bytes
.../load_data/images/time_axis_customize_build.png | Bin 77706 -> 0 bytes
website/docs/modeling/load_data/intro.md | 25 --
.../load_data/segment_operation_settings/intro.md | 94 -----
website/docs/modeling/manual_modeling.md | 331 ----------------
.../modeling/model_design/advance_guide/intro.md | 17 -
.../modeling/model_design/aggregation_group.md | 414 ---------------------
.../docs/modeling/model_design/computed_column.md | 204 ----------
.../modeling/model_design/images/agg/AGG-1.png | Bin 31010 -> 0 bytes
.../model_design/images/agg/advanced-EN-01.png | Bin 205276 -> 0 bytes
.../model_design/images/agg/advanced-EN-02.png | Bin 209971 -> 0 bytes
.../modeling/model_design/images/agg/advanced.png | Bin 110451 -> 0 bytes
.../model_design/images/agg/agg-group-1.png | Bin 183833 -> 0 bytes
.../model_design/images/agg/agg-group-2.png | Bin 100574 -> 0 bytes
.../model_design/images/agg/agg-group-3.png | Bin 61438 -> 0 bytes
.../model_design/images/agg/agg-group-4.png | Bin 66509 -> 0 bytes
.../modeling/model_design/images/agg/agg_1.png | Bin 75878 -> 0 bytes
.../modeling/model_design/images/agg/agg_2.png | Bin 78718 -> 0 bytes
.../model_design/images/agg/agg_index_2.png | Bin 28858 -> 0 bytes
.../model_design/images/agg/agg_measure.png | Bin 65171 -> 0 bytes
.../images/auto_modeling/convert_no.en.png | Bin 147334 -> 0 bytes
.../images/auto_modeling/convert_or_not.en.png | Bin 347569 -> 0 bytes
.../images/auto_modeling/convert_yes_model.en.png | Bin 150441 -> 0 bytes
.../auto_modeling/convert_yes_recommend_en.png | Bin 166435 -> 0 bytes
.../images/auto_modeling/import_sql.en.png | Bin 358875 -> 0 bytes
.../model_design/images/computed_column/cc_1.png | Bin 96616 -> 0 bytes
.../model_design/images/computed_column/cc_11.png | Bin 94064 -> 0 bytes
.../model_design/images/computed_column/cc_2.png | Bin 25693 -> 0 bytes
.../model_design/images/computed_column/cc_3.png | Bin 92496 -> 0 bytes
.../model_design/images/computed_column/cc_4.png | Bin 28699 -> 0 bytes
.../model_design/images/computed_column/cc_5.png | Bin 29304 -> 0 bytes
.../model_design/images/computed_column/cc_6.png | Bin 122044 -> 0 bytes
.../model_design/images/computed_column/cc_8.png | Bin 26076 -> 0 bytes
.../images/count_distinct/cd_measures.png | Bin 128445 -> 0 bytes
.../images/count_distinct/cd_measures_add.1.png | Bin 73495 -> 0 bytes
.../images/count_distinct/cd_measures_add.2.png | Bin 99761 -> 0 bytes
.../images/count_distinct/cd_measures_add.3.png | Bin 120209 -> 0 bytes
.../images/count_distinct/cd_measures_add.4.png | Bin 33140 -> 0 bytes
.../images/count_distinct/cd_measures_add.5.png | Bin 174009 -> 0 bytes
.../images/count_distinct/cd_measures_add.7.png | Bin 12909 -> 0 bytes
.../images/count_distinct/cd_measures_add.8.png | Bin 10685 -> 0 bytes
.../images/count_distinct/cd_measures_add.9.png | Bin 10209 -> 0 bytes
.../images/count_distinct/cd_measures_add.png | Bin 101130 -> 0 bytes
.../count_distinct/cd_measures_add_precisely.png | Bin 88993 -> 0 bytes
.../images/count_distinct/cd_measures_edit.png | Bin 99761 -> 0 bytes
.../images/count_distinct/cd_meausres_add.6.png | Bin 10100 -> 0 bytes
.../images/count_distinct/wd_datasample.png | Bin 68299 -> 0 bytes
.../modeling/model_design/images/import_sql.en.jpg | Bin 359231 -> 0 bytes
.../modeling/model_design/images/mdc/intro_mdc.png | Bin 90718 -> 0 bytes
.../model_design/images/mdc/single_mdc.png | Bin 100541 -> 0 bytes
.../modeling/model_design/images/mdc/total_mdc.png | Bin 89560 -> 0 bytes
.../images/model_check/24_model_diagnose_1.png | Bin 126615 -> 0 bytes
.../images/model_check/24_model_diagnose_2.png | Bin 102317 -> 0 bytes
.../images/model_check/24_model_diagnose_4.png | Bin 108428 -> 0 bytes
.../images/model_check/24_model_diagnose_6.png | Bin 89494 -> 0 bytes
.../images/model_check/25_model_check.png | Bin 99835 -> 0 bytes
.../images/model_check/25_model_save.png | Bin 121755 -> 0 bytes
.../model_design/images/model_design/add_table.png | Bin 139656 -> 0 bytes
.../model_design/images/model_design/dimension.png | Bin 102709 -> 0 bytes
.../images/model_design/dimension_2.png | Bin 69815 -> 0 bytes
.../model_design/images/model_design/join.png | Bin 106888 -> 0 bytes
.../model_design/images/model_design/measure.png | Bin 63310 -> 0 bytes
.../model_design/images/model_design/measure_2.png | Bin 63618 -> 0 bytes
.../model_design/images/model_design/measure_3.png | Bin 252965 -> 0 bytes
.../images/model_design/model_edit_page.png | Bin 260928 -> 0 bytes
.../model_design/model_filter_condition.en.png | Bin 71666 -> 0 bytes
.../images/model_design/model_result.png | Bin 156227 -> 0 bytes
.../images/model_design/partition_en.png | Bin 93138 -> 0 bytes
.../images/model_design/sync_hive_comment_en.png | Bin 504179 -> 0 bytes
.../images/percentile_approximate/1.en.png | Bin 259223 -> 0 bytes
.../percentile_approximate/cube_query.en.png | Bin 227126 -> 0 bytes
.../percentile_approximate/return_type.en.png | Bin 85870 -> 0 bytes
.../modeling/model_design/images/review_model.png | Bin 131496 -> 0 bytes
.../model_design/images/table_index/index_1.png | Bin 121401 -> 0 bytes
.../images/table_index/table_index_1.png | Bin 54536 -> 0 bytes
.../images/table_index/table_index_2.png | Bin 169186 -> 0 bytes
.../images/table_index/table_index_3.png | Bin 305850 -> 0 bytes
.../images/table_index/table_index_disable.png | Bin 85867 -> 0 bytes
.../images/table_index/table_index_enable.png | Bin 364900 -> 0 bytes
.../images/table_index/table_index_encode.png | Bin 230346 -> 0 bytes
.../images/table_index/table_index_index.png | Bin 248221 -> 0 bytes
.../images/table_index/table_index_sortby.png | Bin 273821 -> 0 bytes
.../docs/modeling/model_design/images/topN_1.png | Bin 10172 -> 0 bytes
.../model_design/images/topN_measure_edit.png | Bin 102494 -> 0 bytes
website/docs/modeling/model_design/intro.md | 17 -
website/docs/modeling/model_design/table_index.md | 68 ----
website/docs/modeling/model_management.md | 137 -------
website/docs/operations/job-monitoring/intro.md | 2 +-
.../job-monitoring/job_concept_settings.md | 4 +-
.../job-monitoring/job_exception_resolve.md | 2 +-
website/docs/overview.md | 4 +-
website/docs/query/{ => history}/query_history.md | 16 +-
.../history}/query_history_fields.md | 8 +-
website/docs/quickstart/intro.md | 18 +-
website/docs/quickstart/tutorial.md | 14 +-
website/docs/restapi/error_code.md | 2 +-
website/docs/restapi/model_api/intro.md | 8 +-
website/sidebars.ts | 190 ++++------
385 files changed, 2038 insertions(+), 1994 deletions(-)
diff --git a/website/docs/datasource/data_sampling.md b/website/docs/datasource/data_sampling.md
index ea80a54858..87c4567d09 100644
--- a/website/docs/datasource/data_sampling.md
+++ b/website/docs/datasource/data_sampling.md
@@ -14,7 +14,7 @@ last_update:
date: 08/19/2022
---
-Kylin provides the data sampling function to facilitate table data analysis. With data sampling, you can collect table characteristics, such as cardinality, max value, and min value for each column, to improve [model designing](../modeling/intro.md).
+Kylin provides the data sampling function to facilitate table data analysis. With data sampling, you can collect table characteristics, such as cardinality, max value, and min value for each column, to improve [model designing](../model/intro.md).
## Known limitations
@@ -70,5 +70,5 @@ For more information about commands, see [ALTER TABLE Statement](https://dev.mys
### See also
-[Model](../modeling/intro.md)
+[Model](../model/intro.md)
diff --git a/website/docs/datasource/import_hive.md b/website/docs/datasource/import_hive.md
index 9a601bc0ca..32e7f7fc84 100755
--- a/website/docs/datasource/import_hive.md
+++ b/website/docs/datasource/import_hive.md
@@ -20,7 +20,7 @@ Leveraging the capability of Apache Hive, Kylin has effectively solved the query
Apache Hive is a distributed, fault-tolerant data warehouse software that enables analytics at a massive scale. With Apache Hive to map structured data into tables and precomputation offered by Kylin, you can easily identify and manage your business' most valuable data, and uncover new insights from any size dataset, at any time, and from anywhere.
-This article introduces how to load Hive data source to a Kylin [project](../operations/project-managing/project_management.md) for [model designing](../modeling/intro.md) and [data analysis](../query/insight).
+This article introduces how to load Hive data source to a Kylin [project](../operations/project-managing/project_management.md) for [model designing](../model/intro.md) and [data analysis](../query/insight).
### Notes
@@ -71,5 +71,5 @@ This article introduces how to load Hive data source to a Kylin [project](../ope
### Next step
-[Model](../modeling/intro.md)
+[Model](../model/intro.md)
diff --git a/website/docs/icons/add_icon.svg b/website/docs/icons/add_icon.svg
new file mode 100644
index 0000000000..21b14fdc6c
--- /dev/null
+++ b/website/docs/icons/add_icon.svg
@@ -0,0 +1,5 @@
+<svg xmlns="http://www.w3.org/2000/svg"
+ width="16" height="16" viewBox="0 0 16 16" fill="none" version="1.1">
+ <path d="M8 2v12M2 8h12" stroke="black" fill="none" stroke-width="1"/>
+</svg>
+
diff --git a/website/docs/icons/batch_add_icon.svg b/website/docs/icons/batch_add_icon.svg
new file mode 100644
index 0000000000..42d1c9d567
--- /dev/null
+++ b/website/docs/icons/batch_add_icon.svg
@@ -0,0 +1,4 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 1024 1024" version="1.1">
+ <path d="M789.333333 106.666667a117.333333 117.333333 0 0 1 117.248 112.618666l0.085334 4.714667v405.333333a117.333333 117.333333 0 0 1-112.618667 117.248L789.333333 746.666667h-10.666666v42.666666a85.333333 85.333333 0 0 1-85.333334 85.333334h-469.333333a85.333333 85.333333 0 0 1-85.333333-85.333334V320a85.333333 85.333333 0 0 1 85.333333-85.333333h42.666667v-10.666667a117.333333 117.333333 0 0 1 112.618666-117.248L384 106.666667h405.333333z m-96 192h-469.333333a21.333333 21.333333 [...]
+ fill="#40434E"></path>
+</svg>
diff --git a/website/docs/icons/batch_remove_icon.svg b/website/docs/icons/batch_remove_icon.svg
new file mode 100644
index 0000000000..ba68171c7f
--- /dev/null
+++ b/website/docs/icons/batch_remove_icon.svg
@@ -0,0 +1,9 @@
+<svg xmlns="http://www.w3.org/2000/svg"
+ viewBox="0 0 1024 1024"
+ version="1.1"
+ width="16"
+ height="16">
+ <path d="M786.285714 201.142857h-694.857143a36.571429 36.571429 0 0 0-36.571428 36.571429v694.857143a36.571429 36.571429 0 0 0 36.571428 36.571428h694.857143a36.571429 36.571429 0 0 0 36.571429-36.571428v-694.857143a36.571429 36.571429 0 0 0-36.571429-36.571429z m-45.714285 685.714286h-603.428572v-603.428572h603.428572v603.428572zM932.571429 54.857143H228.571429A9.142857 9.142857 0 0 0 219.428571 64v64c0 5.046857 4.096 9.142857 9.142858 9.142857h658.285714v658.285714c0 5.046857 4.096 [...]
+ fill="#565757">
+ </path>
+</svg>
diff --git a/website/docs/icons/build_icon.svg b/website/docs/icons/build_icon.svg
new file mode 100644
index 0000000000..039d71072f
--- /dev/null
+++ b/website/docs/icons/build_icon.svg
@@ -0,0 +1,5 @@
+<svg class="icon" viewBox="0 0 1024 1024" version="1.1" xmlns="http://www.w3.org/2000/svg"
+ width="16" height="16">
+ <path d="M924.8 236.8L544 19.2c-19.2-12.8-44.8-12.8-64 0L99.2 236.8c-19.2 12.8-32 32-32 54.4v438.4c0 22.4 12.8 44.8 32 54.4L480 1004.8c9.6 6.4 22.4 9.6 32 9.6s22.4-3.2 32-9.6l380.8-217.6c19.2-12.8 32-32 32-54.4V294.4c0-25.6-12.8-44.8-32-57.6z m-32 492.8L512 950.4 131.2 729.6V294.4L512 73.6l380.8 217.6v438.4z"></path>
+ <path d="M480 243.2l-185.6 105.6c-19.2 12.8-32 32-32 54.4v214.4c0 22.4 12.8 44.8 32 54.4l185.6 105.6c9.6 6.4 22.4 9.6 32 9.6s22.4-3.2 32-9.6l185.6-105.6c19.2-12.8 32-32 32-54.4v-214.4c0-22.4-12.8-44.8-32-54.4L544 243.2c-19.2-12.8-44.8-12.8-64 0zM310.4 646.4c-9.6-6.4-16-16-16-28.8v-214.4l201.6 115.2v233.6l-185.6-105.6z m403.2 0l-185.6 105.6v-233.6l201.6-115.2v214.4c0 12.8-6.4 22.4-16 28.8z m0-268.8L512 492.8l-201.6-115.2 185.6-105.6c6.4-3.2 9.6-3.2 16-3.2s9.6 0 16 3.2l185.6 105.6z"></path>
+</svg>
diff --git a/website/docs/icons/circle_add_icon2.svg b/website/docs/icons/circle_add_icon2.svg
new file mode 100644
index 0000000000..16045059ec
--- /dev/null
+++ b/website/docs/icons/circle_add_icon2.svg
@@ -0,0 +1,6 @@
+<svg xmlns="http://www.w3.org/2000/svg"
+ viewBox="0 0 1024 1024"
+ width="10" height="10">
+ <circle cx="650" cy="650" r="350" stroke="black" fill="none" stroke-width="64"/>
+ <path d="M650 325v650M325 650h650" stroke="black" fill="none" stroke-width="64"/>
+</svg>
diff --git a/website/docs/icons/date_search.svg b/website/docs/icons/date_search.svg
new file mode 100644
index 0000000000..396f6250dc
--- /dev/null
+++ b/website/docs/icons/date_search.svg
@@ -0,0 +1,5 @@
+<svg class="icon" viewBox="0 0 1024 1024" version="1.1" xmlns="http://www.w3.org/2000/svg"
+ width="16" height="16">
+ <path d="M341.333333 64a42.666667 42.666667 0 0 1 42.666667 42.666667v21.333333h256V106.666667a42.666667 42.666667 0 1 1 85.333333 0v21.333333h149.333334a85.333333 85.333333 0 0 1 85.333333 85.333333v490.666667a42.666667 42.666667 0 1 1-85.333333 0V448H149.333333v426.666667h362.666667a42.666667 42.666667 0 1 1 0 85.333333H149.333333a85.333333 85.333333 0 0 1-85.333333-85.333333V213.333333a85.333333 85.333333 0 0 1 85.333333-85.333333h149.333334V106.666667a42.666667 42.666667 0 0 1 42 [...]
+ fill="#606060"></path>
+</svg>
diff --git a/website/docs/icons/info_icon.svg b/website/docs/icons/info_icon.svg
new file mode 100644
index 0000000000..477efec990
--- /dev/null
+++ b/website/docs/icons/info_icon.svg
@@ -0,0 +1,7 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 16 16">
+ <circle cx="8" cy="8" r="7" fill="darkgray" />
+ <g transform="rotate(180, 8, 8)">
+ <path d="M8 4v5" stroke="white" stroke-width="2" stroke-linecap="round"/>
+ <circle cx="8" cy="11.5" r="1" fill="white"/>
+ </g>
+</svg>
diff --git a/website/docs/icons/look_details_icon.svg b/website/docs/icons/look_details_icon.svg
new file mode 100644
index 0000000000..4b1f33f93b
--- /dev/null
+++ b/website/docs/icons/look_details_icon.svg
@@ -0,0 +1,5 @@
+<svg class="icon" viewBox="0 0 1024 1024" version="1.1" xmlns="http://www.w3.org/2000/svg"
+ width="16" height="16">
+ <path d="M410.4704 563.2c0 84.48 69.0688 153.6 153.2928 153.6a153.9072 153.9072 0 0 0 153.2928-153.6c0-84.48-68.4544-153.088-152.7296-153.6h-0.512a153.9072 153.9072 0 0 0-153.344 153.6z m496.2304 270.848c19.9168 19.968 19.9168 52.736 0 72.704-10.24 9.728-23.04 14.848-36.352 14.848-13.312 0-26.0608-5.12-36.2496-14.848l-129.2288-129.536a259.584 259.584 0 0 1-141.1072 41.984 255.5904 255.5904 0 0 1-250.3168-204.8H153.6C125.3888 614.4 102.4 591.36 102.4 563.2s22.9888-51.2 51.0976-51.2h15 [...]
+ fill="black"></path>
+</svg>
diff --git a/website/docs/icons/rect_add_icon.svg b/website/docs/icons/rect_add_icon.svg
new file mode 100644
index 0000000000..09ab0e1cce
--- /dev/null
+++ b/website/docs/icons/rect_add_icon.svg
@@ -0,0 +1,4 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 16 16">
+ <path d="M1 1h14v14H1z" stroke="gray" fill="none" stroke-width="1"/>
+ <path d="M8 4v8M4 8h8" stroke="gray" fill="none" stroke-width="1"/>
+</svg>
diff --git a/website/docs/icons/remove_icon.svg b/website/docs/icons/remove_icon.svg
new file mode 100644
index 0000000000..cd00bb6dd6
--- /dev/null
+++ b/website/docs/icons/remove_icon.svg
@@ -0,0 +1,10 @@
+<svg xmlns="http://www.w3.org/2000/svg"
+ viewBox="0 0 1024 1024"
+ version="1.1"
+ width="16"
+ height="16">
+ <path d="M512 1024C229.248 1024 0 794.752 0 512S229.248 0 512 0s512 229.248 512 512-229.248 512-512 512z
+ m0-896C299.968 128 128 299.968 128 512s171.968 384 384 384 384-171.968 384-384S724.032 128 512 128z
+ m192 448H320a64 64 0 1 1 0-128h384a64 64 0 0 1 64 64c0 35.392-28.608 64-64 64z">
+ </path>
+</svg>
diff --git a/website/docs/icons/settings_icon.svg b/website/docs/icons/settings_icon.svg
new file mode 100644
index 0000000000..78bc379eab
--- /dev/null
+++ b/website/docs/icons/settings_icon.svg
@@ -0,0 +1,7 @@
+<svg xmlns="http://www.w3.org/2000/svg"
+ viewBox="0 0 1024 1024"
+ version="1.1"
+ width="16"
+ height="16">
+ <path d="M359.893333 132.202667A64 64 0 0 1 421.546667 85.333333h180.906666a64 64 0 0 1 61.653334 46.869334l21.824 78.549333 78.933333-20.394667a64 64 0 0 1 71.424 29.973334l90.453333 156.672a64 64 0 0 1-9.770666 76.842666L859.882667 512l57.088 58.154667a64 64 0 0 1 9.770666 76.842666l-90.453333 156.672a64 64 0 0 1-71.424 29.973334l-78.933333-20.394667-21.802667 78.549333A64 64 0 0 1 602.453333 938.666667h-180.906666a64 64 0 0 1-61.653334-46.869334l-21.824-78.528-78.933333 20.373334A [...]
+</svg>
diff --git a/website/docs/icons/wizard_icon.svg b/website/docs/icons/wizard_icon.svg
new file mode 100644
index 0000000000..a1bc40d21a
--- /dev/null
+++ b/website/docs/icons/wizard_icon.svg
@@ -0,0 +1,8 @@
+<svg class="icon"
+ viewBox="0 0 1024 1024"
+ version="1.1"
+ xmlns="http://www.w3.org/2000/svg"
+ width="20" height="20">
+ <path d="M773.952 128l25.472 126.976L896 274.24l-96.64 19.328-25.408 126.976-25.408-126.976-96.64-19.328 96.64-19.264L773.952 128zM383.488 371.776l-18.304-54.848-54.912-18.304 54.912-18.24 18.304-54.848 18.304 54.848 54.912 18.24-54.912 18.304-18.304 54.848zM730.048 610.624l19.52 78.08 19.52-78.08 78.08-19.456-78.08-19.52-19.52-78.016-19.52 78.08-78.08 19.456 78.08 19.456zM570.368 349.824l-428.608 428.16a49.664 49.664 0 0 0 0.128 68.736l34.56 34.56a47.68 47.68 0 0 0 68.864 0.064l428. [...]
+ fill="#2F374C"></path>
+</svg>
diff --git a/website/docs/model/features/excluded_column.md b/website/docs/model/features/excluded_column.md
new file mode 100644
index 0000000000..0d7753f439
--- /dev/null
+++ b/website/docs/model/features/excluded_column.md
@@ -0,0 +1,22 @@
+---
+title: Query History Recommendation
+language: en
+sidebar_label: Query History Recommendation
+pagination_label: Query History Recommendation
+toc_min_heading_level: 2
+toc_max_heading_level: 6
+pagination_prev: null
+pagination_next: null
+keywords:
+ - optimization
+draft: false
+last_update:
+ date: 09/18/2024
+---
+
+Exclude Column
+
+ * For join relation, in snowflake model, A join B(on a1 = b1) and B join C (on b2 = c2), the foreign key b2 cannot be a exclude column, otherwise the join chain will be broken when proposing recommendations.
+ * The design of computed columns indicates that precomputation is required, while the design of exclude columns indicates that precomputation is not required. So these two are incompatible with each other. Therefore, it is not recommended to define the computed columns contains exclude columns in principle. In particular, computed columns and measures that dependent on exclude columns may result in incorrect data if they hit the index.
+ * Computed columns dependent on excluded columns are not reused for the intelligent recommendation. Moreover, with the setting of `kylin.metadata.only-reuse-user-defined-computed column=true`(by default false), only the customized computed columns will be used for recommendations, that is to say, no more computed columns started with `CC_AUTO` will be created in the model.
+ * For queries, by default indexes answer the query with a higher priority than using the foreign key and snapshot. After the exclude column is enabled and setting `kylin.query.snapshot-preferred-for-table-exclusion=true`, the system preferentially selects the foreign key index to runtime join snapshot to response the query. When the '**Exclude Column Rule Setting**' is enabled, this is the default behavior. However, by setting it to false, index independent on exclude columns will be pr [...]
diff --git a/website/docs/modeling/model_design/advance_guide/fast_bitmap.md b/website/docs/model/features/fast_bitmap.md
similarity index 76%
rename from website/docs/modeling/model_design/advance_guide/fast_bitmap.md
rename to website/docs/model/features/fast_bitmap.md
index 6d9ccf7075..6876759178 100644
--- a/website/docs/modeling/model_design/advance_guide/fast_bitmap.md
+++ b/website/docs/model/features/fast_bitmap.md
@@ -1,8 +1,8 @@
---
-title: Query Optimization for Exact Hit Index
+title: Fast Bitmap Index
language: en
-sidebar_label: Query Optimization for Exact Hit Index
-pagination_label: Query Optimization for Exact Hit Index
+sidebar_label: Fast Bitmap Index
+pagination_label: Fast Bitmap Index
toc_min_heading_level: 2
toc_max_heading_level: 6
pagination_prev: null
@@ -44,31 +44,31 @@ GROUP BY LINEITEM.L_SHIPDATE
1. Create the model:
- 
+ 
2. Switch to the **Model Settings** interface:
- 
+ 
3. Enter the configuration to enable the function:
- 
+ 
4. Add indexes:
- 
+ 
5. After building successfully, the query performance is improved a lot when the query exactly matches index.
- 
+ 
- 
+ 
6. Compare the execution plans before and after optimization
- 
+ 
- 
+ 
### Known Limitations
diff --git a/website/docs/modeling/model_design/advance_guide/images/add_index.png b/website/docs/model/features/images/fast_bitmap/add_index.png
similarity index 100%
rename from website/docs/modeling/model_design/advance_guide/images/add_index.png
rename to website/docs/model/features/images/fast_bitmap/add_index.png
diff --git a/website/docs/modeling/model_design/advance_guide/images/model.png b/website/docs/model/features/images/fast_bitmap/model.png
similarity index 100%
rename from website/docs/modeling/model_design/advance_guide/images/model.png
rename to website/docs/model/features/images/fast_bitmap/model.png
diff --git a/website/docs/modeling/model_design/advance_guide/images/model_config_1.png b/website/docs/model/features/images/fast_bitmap/model_config_1.png
similarity index 100%
rename from website/docs/modeling/model_design/advance_guide/images/model_config_1.png
rename to website/docs/model/features/images/fast_bitmap/model_config_1.png
diff --git a/website/docs/modeling/model_design/advance_guide/images/model_config_2.png b/website/docs/model/features/images/fast_bitmap/model_config_2.png
similarity index 100%
rename from website/docs/modeling/model_design/advance_guide/images/model_config_2.png
rename to website/docs/model/features/images/fast_bitmap/model_config_2.png
diff --git a/website/docs/modeling/model_design/advance_guide/images/query_new.png b/website/docs/model/features/images/fast_bitmap/query_new.png
similarity index 100%
rename from website/docs/modeling/model_design/advance_guide/images/query_new.png
rename to website/docs/model/features/images/fast_bitmap/query_new.png
diff --git a/website/docs/modeling/model_design/advance_guide/images/query_old.png b/website/docs/model/features/images/fast_bitmap/query_old.png
similarity index 100%
rename from website/docs/modeling/model_design/advance_guide/images/query_old.png
rename to website/docs/model/features/images/fast_bitmap/query_old.png
diff --git a/website/docs/modeling/model_design/advance_guide/images/spark_plan_new.png b/website/docs/model/features/images/fast_bitmap/spark_plan_new.png
similarity index 100%
rename from website/docs/modeling/model_design/advance_guide/images/spark_plan_new.png
rename to website/docs/model/features/images/fast_bitmap/spark_plan_new.png
diff --git a/website/docs/modeling/model_design/advance_guide/images/spark_plan_old.png b/website/docs/model/features/images/fast_bitmap/spark_plan_old.png
similarity index 100%
rename from website/docs/modeling/model_design/advance_guide/images/spark_plan_old.png
rename to website/docs/model/features/images/fast_bitmap/spark_plan_old.png
diff --git a/website/docs/modeling/model_design/measure_design/images/add_collect_set.png b/website/docs/model/features/images/measures/collect_set/add_collect_set.png
similarity index 100%
rename from website/docs/modeling/model_design/measure_design/images/add_collect_set.png
rename to website/docs/model/features/images/measures/collect_set/add_collect_set.png
diff --git a/website/docs/modeling/model_design/measure_design/images/collect_result.png b/website/docs/model/features/images/measures/collect_set/collect_result.png
similarity index 100%
rename from website/docs/modeling/model_design/measure_design/images/collect_result.png
rename to website/docs/model/features/images/measures/collect_set/collect_result.png
diff --git a/website/docs/modeling/model_design/measure_design/images/concatws_result.png b/website/docs/model/features/images/measures/collect_set/concatws_result.png
similarity index 100%
rename from website/docs/modeling/model_design/measure_design/images/concatws_result.png
rename to website/docs/model/features/images/measures/collect_set/concatws_result.png
diff --git a/website/docs/modeling/model_design/measure_design/images/corr_add_measure.png b/website/docs/model/features/images/measures/corr/corr_add_measure.png
similarity index 100%
rename from website/docs/modeling/model_design/measure_design/images/corr_add_measure.png
rename to website/docs/model/features/images/measures/corr/corr_add_measure.png
diff --git a/website/docs/modeling/model_design/measure_design/images/corr_edit_measure.png b/website/docs/model/features/images/measures/corr/corr_edit_measure.png
similarity index 100%
rename from website/docs/modeling/model_design/measure_design/images/corr_edit_measure.png
rename to website/docs/model/features/images/measures/corr/corr_edit_measure.png
diff --git a/website/docs/modeling/model_design/measure_design/images/corr_query.png b/website/docs/model/features/images/measures/corr/corr_query.png
similarity index 100%
rename from website/docs/modeling/model_design/measure_design/images/corr_query.png
rename to website/docs/model/features/images/measures/corr/corr_query.png
diff --git a/website/docs/modeling/model_design/measure_design/images/cd_measures_add_precisely.png b/website/docs/model/features/images/measures/count_distinct_bitmap/cd_measures_add_precisely.png
similarity index 100%
rename from website/docs/modeling/model_design/measure_design/images/cd_measures_add_precisely.png
rename to website/docs/model/features/images/measures/count_distinct_bitmap/cd_measures_add_precisely.png
diff --git a/website/docs/modeling/model_design/measure_design/images/cd_measures_edit.png b/website/docs/model/features/images/measures/count_distinct_hllc/cd_measures_edit.png
similarity index 100%
rename from website/docs/modeling/model_design/measure_design/images/cd_measures_edit.png
rename to website/docs/model/features/images/measures/count_distinct_hllc/cd_measures_edit.png
diff --git a/website/docs/modeling/model_design/measure_design/images/percentile_approximate.png b/website/docs/model/features/images/measures/percentile/percentile_approximate.png
similarity index 100%
rename from website/docs/modeling/model_design/measure_design/images/percentile_approximate.png
rename to website/docs/model/features/images/measures/percentile/percentile_approximate.png
diff --git a/website/docs/modeling/model_design/measure_design/images/percentile_result.png b/website/docs/model/features/images/measures/percentile/percentile_result.png
similarity index 100%
rename from website/docs/modeling/model_design/measure_design/images/percentile_result.png
rename to website/docs/model/features/images/measures/percentile/percentile_result.png
diff --git a/website/docs/modeling/model_design/measure_design/images/percentile_result_hive.png b/website/docs/model/features/images/measures/percentile/percentile_result_hive.png
similarity index 100%
rename from website/docs/modeling/model_design/measure_design/images/percentile_result_hive.png
rename to website/docs/model/features/images/measures/percentile/percentile_result_hive.png
diff --git a/website/docs/modeling/model_design/measure_design/images/topN_measure_edit.png b/website/docs/model/features/images/measures/topn/topN_measure_edit.png
similarity index 100%
rename from website/docs/modeling/model_design/measure_design/images/topN_measure_edit.png
rename to website/docs/model/features/images/measures/topn/topN_measure_edit.png
diff --git a/website/docs/modeling/model_design/measure_design/images/topn_result.png b/website/docs/model/features/images/measures/topn/topn_result.png
similarity index 100%
rename from website/docs/modeling/model_design/measure_design/images/topn_result.png
rename to website/docs/model/features/images/measures/topn/topn_result.png
diff --git a/website/docs/modeling/model_design/advance_guide/images/multilevel_partioning_build_subp_first.png b/website/docs/model/features/images/multi_partitioning/multilevel_partioning_build_subp_first.png
similarity index 100%
rename from website/docs/modeling/model_design/advance_guide/images/multilevel_partioning_build_subp_first.png
rename to website/docs/model/features/images/multi_partitioning/multilevel_partioning_build_subp_first.png
diff --git a/website/docs/modeling/model_design/advance_guide/images/multilevel_partioning_build_subp_second.png b/website/docs/model/features/images/multi_partitioning/multilevel_partioning_build_subp_second.png
similarity index 100%
rename from website/docs/modeling/model_design/advance_guide/images/multilevel_partioning_build_subp_second.png
rename to website/docs/model/features/images/multi_partitioning/multilevel_partioning_build_subp_second.png
diff --git a/website/docs/modeling/model_design/advance_guide/images/multilevel_partioning_build_subp_second_0.png b/website/docs/model/features/images/multi_partitioning/multilevel_partioning_build_subp_second_0.png
similarity index 100%
rename from website/docs/modeling/model_design/advance_guide/images/multilevel_partioning_build_subp_second_0.png
rename to website/docs/model/features/images/multi_partitioning/multilevel_partioning_build_subp_second_0.png
diff --git a/website/docs/modeling/model_design/advance_guide/images/multilevel_partioning_build_subp_second_1.png b/website/docs/model/features/images/multi_partitioning/multilevel_partioning_build_subp_second_1.png
similarity index 100%
rename from website/docs/modeling/model_design/advance_guide/images/multilevel_partioning_build_subp_second_1.png
rename to website/docs/model/features/images/multi_partitioning/multilevel_partioning_build_subp_second_1.png
diff --git a/website/docs/modeling/model_design/advance_guide/images/multilevel_partitioning_close.png b/website/docs/model/features/images/multi_partitioning/multilevel_partitioning_close.png
similarity index 100%
rename from website/docs/modeling/model_design/advance_guide/images/multilevel_partitioning_close.png
rename to website/docs/model/features/images/multi_partitioning/multilevel_partitioning_close.png
diff --git a/website/docs/modeling/model_design/advance_guide/images/multilevel_partitioning_model_save.png b/website/docs/model/features/images/multi_partitioning/multilevel_partitioning_model_save.png
similarity index 100%
rename from website/docs/modeling/model_design/advance_guide/images/multilevel_partitioning_model_save.png
rename to website/docs/model/features/images/multi_partitioning/multilevel_partitioning_model_save.png
diff --git a/website/docs/modeling/model_design/advance_guide/images/multilevel_partitioning_query.png b/website/docs/model/features/images/multi_partitioning/multilevel_partitioning_query.png
similarity index 100%
rename from website/docs/modeling/model_design/advance_guide/images/multilevel_partitioning_query.png
rename to website/docs/model/features/images/multi_partitioning/multilevel_partitioning_query.png
diff --git a/website/docs/modeling/model_design/advance_guide/images/multilevel_partitioning_set.png b/website/docs/model/features/images/multi_partitioning/multilevel_partitioning_set.png
similarity index 100%
rename from website/docs/modeling/model_design/advance_guide/images/multilevel_partitioning_set.png
rename to website/docs/model/features/images/multi_partitioning/multilevel_partitioning_set.png
diff --git a/website/docs/modeling/model_design/advance_guide/images/multilevel_partitioning_subp_value.png b/website/docs/model/features/images/multi_partitioning/multilevel_partitioning_subp_value.png
similarity index 100%
rename from website/docs/modeling/model_design/advance_guide/images/multilevel_partitioning_subp_value.png
rename to website/docs/model/features/images/multi_partitioning/multilevel_partitioning_subp_value.png
diff --git a/website/docs/modeling/model_design/images/scd2/historical_dimension_table_switch.png b/website/docs/model/features/images/scd2/historical_dimension_table_switch.png
similarity index 100%
rename from website/docs/modeling/model_design/images/scd2/historical_dimension_table_switch.png
rename to website/docs/model/features/images/scd2/historical_dimension_table_switch.png
diff --git a/website/docs/modeling/model_design/images/model_SCD1.png b/website/docs/model/features/images/scd2/model_SCD1.png
similarity index 100%
rename from website/docs/modeling/model_design/images/model_SCD1.png
rename to website/docs/model/features/images/scd2/model_SCD1.png
diff --git a/website/docs/modeling/model_design/images/model_SCD2.png b/website/docs/model/features/images/scd2/model_SCD2.png
similarity index 100%
rename from website/docs/modeling/model_design/images/model_SCD2.png
rename to website/docs/model/features/images/scd2/model_SCD2.png
diff --git a/website/docs/modeling/model_design/images/scd2/model_SCD2_5x.png b/website/docs/model/features/images/scd2/model_SCD2_5x.png
similarity index 100%
rename from website/docs/modeling/model_design/images/scd2/model_SCD2_5x.png
rename to website/docs/model/features/images/scd2/model_SCD2_5x.png
diff --git a/website/docs/modeling/model_design/images/scd2/model_historical_dimension_table_scd2.png b/website/docs/model/features/images/scd2/model_historical_dimension_table_scd2.png
similarity index 100%
rename from website/docs/modeling/model_design/images/scd2/model_historical_dimension_table_scd2.png
rename to website/docs/model/features/images/scd2/model_historical_dimension_table_scd2.png
diff --git a/website/docs/modeling/model_design/images/scd2/model_historical_dimension_table_scd2_join.png b/website/docs/model/features/images/scd2/model_historical_dimension_table_scd2_join.png
similarity index 100%
rename from website/docs/modeling/model_design/images/scd2/model_historical_dimension_table_scd2_join.png
rename to website/docs/model/features/images/scd2/model_historical_dimension_table_scd2_join.png
diff --git a/website/docs/modeling/model_design/advance_guide/integer_encoding.md b/website/docs/model/features/integer_encoding.md
similarity index 82%
rename from website/docs/modeling/model_design/advance_guide/integer_encoding.md
rename to website/docs/model/features/integer_encoding.md
index d7dfbbdb79..79ea077bc0 100644
--- a/website/docs/modeling/model_design/advance_guide/integer_encoding.md
+++ b/website/docs/model/features/integer_encoding.md
@@ -1,8 +1,8 @@
---
-title: Skip Dictionary Encoding Optimization for Integer Type
+title: Dictionary Encoding
language: en
-sidebar_label: Skip Dictionary Encoding Optimization for Integer Type
-pagination_label: Skip Dictionary Encoding Optimization for Integer Type
+sidebar_label: Dictionary Encoding
+pagination_label: Dictionary Encoding
toc_min_heading_level: 2
toc_max_heading_level: 6
pagination_prev: null
diff --git a/website/docs/modeling/model_design/measure_design/collect_set.md b/website/docs/model/features/measures/collect_set.md
similarity index 93%
rename from website/docs/modeling/model_design/measure_design/collect_set.md
rename to website/docs/model/features/measures/collect_set.md
index 8e1e03cc9a..42037f971c 100644
--- a/website/docs/modeling/model_design/measure_design/collect_set.md
+++ b/website/docs/model/features/measures/collect_set.md
@@ -26,9 +26,9 @@ Let’s use the project created in the chapter [Tutorial](../../../quickstart/tu
We will use the fact table `SSB.P_LINEORDER`. This sample table is a mockup of transactions that can happen in an online marketplace. It has a couple of dimension and measure columns. For easy understanding, we will only use two columns: `LO_CUSTKEY` and `LO_ORDERDATE`. The table below gives an introduction of these columns.
| Column | Description |
-| :----------- | :---------- |
+|:-------------|:------------|
| LO_CUSTKEY | Customer ID |
-| LO_ORDERDATE | Order Date |
+| LO_ORDERDATE | Order Date |
We want to query the order date combination of each customer. The order date will be returned as an array with deduplicated values. The query example is below:
@@ -46,13 +46,13 @@ Before the **COLLECT_SET** measure is added, the query will pushdown to Hive. Ac
Please add a measure in the model editing page as follows. Please fill in the measure **Name**, for example `COLLECT_SET_ORDERDATE`, select **Function** as **COLLECT_SET**. Finally select the target column from the dropdown list, for example `P_LINEORDER.LO_ORDERDATE`.
-
+
Once the measure is added and the model is saved, click **Add Index** in the pop-up window to enter the **Model Index** page. You need to click **+**（Add Aggregate Group) under the **Aggregate Group** tab, add the corresponding dimensions and measures to the appropriate aggregate group according to your business scenario, and the new aggregate index will be generated after submission. In this example, the new index will contain the dimension `LO_CUSTKEY` and the measure ` COLLECT_SET(LO_ [...]
Resubmit the above SQL query in the **Query -> Insight** page, and you will find the result as below:
-
+
If you need to create a model from the very beginning and add a COLLECT_SET measure, please add some indexes and load data into the model. A model won't be able to serve any query if it has no index and data. You can read this chapter [Model Design Basics](../../intro.md) to understand the method of model design.
@@ -64,6 +64,6 @@ FROM SSB.P_LINEORDER
GROUP BY LO_CUSTKEY
```
-
+
> **Note**: The CONCAT_WS function is only supported in conjunction with the COLLECT_SET function when querying.
diff --git a/website/docs/modeling/model_design/measure_design/corr.md b/website/docs/model/features/measures/corr.md
similarity index 92%
rename from website/docs/modeling/model_design/measure_design/corr.md
rename to website/docs/model/features/measures/corr.md
index fb6e2e9d01..89c492b61e 100644
--- a/website/docs/modeling/model_design/measure_design/corr.md
+++ b/website/docs/model/features/measures/corr.md
@@ -38,15 +38,15 @@ FROM SSB.P_LINEORDER
Step 1, in the model edit view, click *Add* in the *Measure* sheet
-
+
Step 2, select the *CORR* function and corresponding columns
-
+
Step 3, after saving the model and building the indexes, users can do queries in the *Insight* page
-
+
### Notes
1. The *CORR* measure may need to create a bunch of internal measuers (which are hidden from the end user) and computed columns on the model. Kylin will create and manage those internal measures itself underneath. If you wish to delete the auto generated computed columns, you should remove the corresponding corr measure first.
diff --git a/website/docs/modeling/model_design/measure_design/count_distinct_bitmap.md b/website/docs/model/features/measures/count_distinct_bitmap.md
similarity index 97%
rename from website/docs/modeling/model_design/measure_design/count_distinct_bitmap.md
rename to website/docs/model/features/measures/count_distinct_bitmap.md
index c64b89be4a..54a3ae05ed 100644
--- a/website/docs/modeling/model_design/measure_design/count_distinct_bitmap.md
+++ b/website/docs/model/features/measures/count_distinct_bitmap.md
@@ -37,7 +37,7 @@ Kylin offers both approximate Count Distinct function and precise Count Distinct
> **Note:** The query of precise Count Distinct is based on bitmap, so it will consume more resources.
-
+
Once the measure is added and the model is saved, you need to go to the **Edit Aggregate Index** page, add the corresponding dimensions and measures to the appropriate aggregate group according to your business scenario, and the new aggregate index will be generated after submission. You need to build index and load data to complete the precomputation of the target column. You can check the job of Build Index in the Job Monitor page. After the index is built, you can use the **Count Dist [...]
diff --git a/website/docs/modeling/model_design/measure_design/count_distinct_hllc.md b/website/docs/model/features/measures/count_distinct_hllc.md
similarity index 97%
rename from website/docs/modeling/model_design/measure_design/count_distinct_hllc.md
rename to website/docs/model/features/measures/count_distinct_hllc.md
index ea01fa55dd..e3b1f2ca76 100644
--- a/website/docs/modeling/model_design/measure_design/count_distinct_hllc.md
+++ b/website/docs/model/features/measures/count_distinct_hllc.md
@@ -38,7 +38,7 @@ Before using Count Distinct query, you need to check the target column is ready.
Please add a measure in the model editing page as follows. Please fill in the measure **Name**, such as `DISTINCT_SHIPPRIOTITY`, select **Function** as **COUNT_DISTINCT**, select accuracy requirement from **Function Parameter**, and finally select the target column from the drop-down list.
-
+
Once the measure is added and the model is saved, click **Add Index** in the pop-up window to enter the **Model Index** page. You need to click **+**（Add Aggregate Group) under the **Aggregate Group** tab, add the corresponding dimensions and measures to the appropriate aggregate group according to your business scenario, and the new aggregate index will be generated after submission. You need to build index and load data to complete the precomputation of the target column. You can check [...]
diff --git a/website/docs/modeling/model_design/measure_design/intro.md b/website/docs/model/features/measures/intro.md
similarity index 100%
rename from website/docs/modeling/model_design/measure_design/intro.md
rename to website/docs/model/features/measures/intro.md
diff --git a/website/docs/modeling/model_design/measure_design/percentile_approx.md b/website/docs/model/features/measures/percentile_approx.md
similarity index 87%
rename from website/docs/modeling/model_design/measure_design/percentile_approx.md
rename to website/docs/model/features/measures/percentile_approx.md
index 6efa151cec..26d3fc7539 100644
--- a/website/docs/modeling/model_design/measure_design/percentile_approx.md
+++ b/website/docs/model/features/measures/percentile_approx.md
@@ -27,7 +27,7 @@ Percentile_approx returns the value of below which a given percentage of observa
- `percentile_approx({measure},p,B)`
-*measure* is the measure to query. *p* is number between 0 to 1, inclusive. *B* controls the Approximate Accuracy. The higher the value, the higher the accuracy of the result. percentile_approx uses the interpolation method to determine the value of the nth percentile.
+*measure* is the measure to query. *p* is number between 0 and 1, inclusive. *B* controls the Approximate Accuracy. The higher the value, the higher the accuracy of the result. percentile_approx uses the interpolation method to determine the value of the nth percentile.
@@ -38,7 +38,7 @@ Let’s use the project created in the chapter [Tutorial](../../../quickstart/tu
We will use the fact table `SSB.P_LINEORDER`. This sample table is a mockup of transactions that can happen in an online marketplace. It has a couple of dimension and measure columns. For easy undersatning, we will only use two columns: `LO_SUPPKEY` and `LO_ORDTOTALPRICE`. The table below gives an introduction to these columns.
| Column | Description |
-| ---------------- | ----------- |
+|------------------|-------------|
| LO_SUPPKEY | Supplier ID |
| LO_ORDTOTALPRICE | Sold amount |
@@ -51,7 +51,7 @@ GROUP BY LO_SUPPKEY
```
Before the **PERCENTILE_APPROX** measure is added, the system will pushdown the query to Hive if the query pushdown function is enabled.
-
+
@@ -59,10 +59,10 @@ Before the **PERCENTILE_APPROX** measure is added, the system will pushdown the
Please add a measure in the model editing page as follows. Please fill in the measure **Name** for example `PERCENTILE_ORDTOTALPRICE`, select **Function** as **PERCENTILE_APPROX**, select **Function Parameter** as 'percentile(100)', 'percentile(1000)' or 'percentile(10000)' on demand. The Function Parameter means B listed in the above syntax. Higher value means higher accuracy and higher storage resources are occupied. Finally select the target column from the dropdown list.
-
+
Once the measure is added and the model is saved, click **Add Index** in the pop-up window to enter the **Model Index** page. You need to click **+**（Add Aggregate Group) under the **Aggregate Group** tab, add the dimensions and measures to the aggregate groups according to your business scenario, and the new aggregate indices will be generated after submission. In this example, the new index will contain the dimension `LO_SUPPKEY` and the measure ` percentile_approx(LO_ORDTOTALPRICE, p, [...]
Resubmit the above SQL query in the **Query -> Insight** page, and you will find the result returns the value of the 50th percentage for each supplier's sold amount.
-
+
diff --git a/website/docs/modeling/model_design/measure_design/topn.md b/website/docs/model/features/measures/topn.md
similarity index 97%
rename from website/docs/modeling/model_design/measure_design/topn.md
rename to website/docs/model/features/measures/topn.md
index 6ee45d7e4a..d11446acd4 100644
--- a/website/docs/modeling/model_design/measure_design/topn.md
+++ b/website/docs/model/features/measures/topn.md
@@ -54,11 +54,11 @@ limit 100
It returns multiple records. See below:
-
+
*Method 2*: In order to get the desired query performance on a massive dataset, we recommend creating a Top-N measure for the target column and pre-calculating it when building the index. Please add a measure in the model editing page as follows. Fill in the measure **Name** for example `TOTAL_AMOUNT`, select **Function** as **TOP_N**, select **Function Parameter** as **Top 100**, and finally select the target column from the dropdown list.
-
+
Once the measure is added and the model is saved, you need to go to the **Edit Aggregate Index** page, add the corresponding dimensions and measures to the appropriate aggregate group according to your business scenario, and the new aggregate index will be generated after submission. In this example, the new index will contain the dimension `LO_ORDERDATE`, `LO_SUPPKEY`, `LO_PARTKEY` and the measure ` SUM(LO_ORDTOTALPRICE) `, you need to build index and load data to complete the precomput [...]
diff --git a/website/docs/modeling/model_design/advance_guide/multilevel_partitioning.md b/website/docs/model/features/multi_partitioning.md
similarity index 90%
rename from website/docs/modeling/model_design/advance_guide/multilevel_partitioning.md
rename to website/docs/model/features/multi_partitioning.md
index 56d9716c20..596257155f 100644
--- a/website/docs/modeling/model_design/advance_guide/multilevel_partitioning.md
+++ b/website/docs/model/features/multi_partitioning.md
@@ -1,8 +1,8 @@
---
-title: Multi-Level Partitioning (Beta)
+title: Multilevel Partitioning
language: en
-sidebar_label: Multi-Level Partitioning (Beta)
-pagination_label: Multi-Level Partitioning (Beta)
+sidebar_label: Multilevel Partitioning
+pagination_label: Multilevel Partitioning
toc_min_heading_level: 2
toc_max_heading_level: 6
pagination_prev: null
@@ -29,18 +29,18 @@ Kylin multi-level partition currently only supports two level partition.
You can enable or disable support for multi-level partitioning in the model **Settings-Advanced Settings-Multi-level Partitioning**.
-
+
**Note:** When the multi-level partitioning of the model is closed, the model using the multi-level partitioning will be automatically offline. If you need to go online, you need to delete the sub-partition before going online or turn on this option.
-
+
### <span id="set">Set model sub-partition</span>
After enabling multi-level partitioning, you can create a model in **Data Asset- Model- +Model**. Then you can choose to add sub-partition columns when saving the model, and the types currently supported as sub-partition columns are **tinyint,smallint,int/integer,bigint,double,decimal,timestamp,date,varchar,char,boolean**.
-
+
Or you can adjust the sub-partition column in **Model List-...(More Actions)-Model Partition**.
@@ -52,7 +52,7 @@ You can also add, delete or search for sub-partition values in **Model List-...(
When adding sub-partition values, the system does not check the correctness. The system allows adding sub-partition values that do not yet exist. When querying, the sub-partition value must be exactly the same with the sub-partition value to match (case sensitive, wildcards matching is not supported). Please ensure that the added sub-partition value meets your expectations.
-
+
@@ -60,15 +60,15 @@ When adding sub-partition values, the system does not check the correctness. The
When constructing a Segment of a new time range, you can click **Model List-Build Index** and specify the sub-partition value during construction
-
+
When the segment already exists, but only some sub-partitions under the segment have been constructed, you can click the model name to enter the model information page. You can click **Segment** to view the constructed sub-partitions in **Subpartition**, or continue to build sub-partitions that have not been constructed
-
+
-
+
-
+
If you need to merge Segments, you need to ensure that the sub-partition values are consistent.
@@ -99,7 +99,7 @@ The following common cases help understand. Suppose there are 4 Segments in the
- Segment 4, the time range is [2018-2019), does not exist in the system, for ease of understanding, the code is Segment 4
- Segment 5, the time range is [2019-2020), reserved Segment, does not include subpartitions and indexes
-
+
When there are the following modes of query, the system will answer the query in this way:
@@ -153,7 +153,7 @@ Solution: You can adjust MySQL configuration as `max_allowed_packet=256M` to avo
Error Two:
-Prompt: The build fails. The kylin.log log prompts `The size of BLOB/TEXT data inserted in one transaction is greater than 10% of redo log size. Increase the redo log size using innodb_log_file_size`
+Prompt: The build fails. The *kylin.log* prompts `The size of BLOB/TEXT data inserted in one transaction is greater than 10% of redo log size. Increase the redo log size using innodb_log_file_size`
Reason: The amount of data written to mysql redo log in a single transaction exceeds 10% of innodb_log_file_size.
diff --git a/website/docs/modeling/model_design/precompute_join_relations.md b/website/docs/model/features/runtime_join.md
similarity index 97%
rename from website/docs/modeling/model_design/precompute_join_relations.md
rename to website/docs/model/features/runtime_join.md
index 323ff238eb..5030273249 100644
--- a/website/docs/modeling/model_design/precompute_join_relations.md
+++ b/website/docs/model/features/runtime_join.md
@@ -1,8 +1,8 @@
---
-title: Precompute the join relations
+title: Runtime Join
language: en
-sidebar_label: Precompute the join relations
-pagination_label: Precompute the join relations
+sidebar_label: Runtime Join
+pagination_label: Runtime Join
toc_min_heading_level: 2
toc_max_heading_level: 6
pagination_prev: null
diff --git a/website/docs/modeling/auto_modeling/scalar_subquery.md b/website/docs/model/features/scalar_subquery.md
similarity index 91%
rename from website/docs/modeling/auto_modeling/scalar_subquery.md
rename to website/docs/model/features/scalar_subquery.md
index aa1a14946f..8f0dd18d47 100644
--- a/website/docs/modeling/auto_modeling/scalar_subquery.md
+++ b/website/docs/model/features/scalar_subquery.md
@@ -1,8 +1,8 @@
---
-title: Constant Subquery Recommendation
+title: Scalar Subquery
language: en
-sidebar_label: Constant Subquery Recommendation
-pagination_label: Constant Subquery Recommendation
+sidebar_label: Scalar Subquery
+pagination_label: Scalar Subquery
toc_min_heading_level: 2
toc_max_heading_level: 6
pagination_prev: null
@@ -44,4 +44,4 @@ Kylin can not generate an index containing the measure `SUM(D_YEAR)` through rec
### Known Limitation
1. When the aggregate function is `count distinct`, index recommendation and SQL modeling are not currently supported. For example, when `sum(D_YEAR)` becomes `count(distinct D_YEAR)` in the above example, it cannot be recommended;
-2. When the parameter of the aggregate function is an expression, index recommendation and SQL modeling are not currently supported. For example, when `sum(D_YEAR)` becomes `sum(D_YEAR+1)` in the above example, it cannot be recommended.
\ No newline at end of file
+2. When the parameter of the aggregate function is an expression, index recommendation and SQL modeling are not currently supported. For example, when `sum(D_YEAR)` becomes `sum(D_YEAR+1)` in the above example, it cannot be recommended.
diff --git a/website/docs/modeling/model_design/slowly_changing_dimension.md b/website/docs/model/features/scd2.md
similarity index 97%
rename from website/docs/modeling/model_design/slowly_changing_dimension.md
rename to website/docs/model/features/scd2.md
index 85dc952d2c..824fe3f969 100644
--- a/website/docs/modeling/model_design/slowly_changing_dimension.md
+++ b/website/docs/model/features/scd2.md
@@ -1,14 +1,15 @@
---
-title: Slowly Changing Dimension
+title: SCD2
language: en
-sidebar_label: Slowly Changing Dimension
-pagination_label: Slowly Changing Dimension
+sidebar_label: SCD2
+pagination_label: SCD2
toc_min_heading_level: 2
toc_max_heading_level: 6
pagination_prev: null
pagination_next: null
keywords:
- slowly changing dimension
+ - scd2
draft: false
last_update:
date: 08/19/2022
diff --git a/website/docs/model/images/volume_per_time.png b/website/docs/model/images/volume_per_time.png
new file mode 100644
index 0000000000..c106585c22
Binary files /dev/null and b/website/docs/model/images/volume_per_time.png differ
diff --git a/website/docs/model/intro.md b/website/docs/model/intro.md
new file mode 100644
index 0000000000..b2931b444c
--- /dev/null
+++ b/website/docs/model/intro.md
@@ -0,0 +1,68 @@
+---
+title: Model
+language: en
+sidebar_label: Model
+pagination_label: Model
+toc_min_heading_level: 2
+toc_max_heading_level: 6
+pagination_prev: null
+pagination_next: null
+keywords:
+ - model
+draft: false
+last_update:
+ date: 09/14/2024
+---
+
+### Introduction
+
+Kylin utilizes multidimensional modeling theory to build star or snowflake schemas based on tables, making it a powerful tool for large-scale data analysis. The **model** is Kylin's core component, consisting of three key aspects: *model design*, *index design*, and *data loading*. By carefully designing the model, optimizing indexes, and pre-computed data, queries executed on Kylin can avoid scanning the entire dataset, potentially reducing response times to mere seconds, even for petab [...]
+
++ **Model design** refers to establishing relationships between data tables to enable fast extraction of key information from multidimensional data. The core elements of model design are computed columns, dimensions, measures, and join relations.
+
++ **Index design** refers to creating indexes (CUBEs) within the model to precompute query results, thereby reducing query response time. Well-designed indexes not only improve query performance but also help minimize the storage and data-loading costs associated with precomputation.
+
++ **Data loading** refers to the process of importing data into the model, enabling queries to utilize the pre-built indexes rather than scanning the entire dataset. This allows for faster query responses by leveraging the model's optimized structure.
+
+
+
+### Core Concepts
+
+- **Dimension**: A perspective of viewing data, which can be used to describe object attributes or characteristics, for example, product category.
+
+- **Measure**: An aggregated sum, which is usually a continuous value, for example, product sales.
+
+- **Pre-computation**: The process of aggregating data based on model dimension combinations and of storing the results as indexes to accelerate data query.
+
+- **Index**: Also called CUBE, which is used to accelerate data query. Indexes are divided into:
+ - **Aggregate Index**: An aggregated combination of multiple dimensions and measures, and can be used to answer aggregate queries such as total sales for a given year.
+ - **Table Index**: A multilevel index in a wide table and can be used to answer detailed queries such as the last 100 transactions of a certain user.
+
+
+### Why Use Kylin
+
++ **Low Query Latency vs. Large Volume**
+
+ When analyzing massive data, there are some techniques to speed up computing and storage, but they cannot change the time complexity of query, that is, query latency and data volume are linearly dependent.
+
+ If it takes 1 minute to query 100 million entries of data records, querying 10 billion data entries will take about 1 hour and 40 minutes. When companies want to analyze all business data piled up over the years or to add complexity to query, say, with more dimensions, queries will be running extremely slow or even time out.
+
+ 
+
++ **Pre-computation vs. Runtime Computation**
+
+ Pre-computation and runtime computation are two approaches to calculating results in data processing and analytics. **Pre-computation** involves calculating and storing results in advance, so they can be quickly retrieved when a query is run. In contrast, **runtime computation** dynamically computes results during query execution, processing raw data and applying aggregations, filters, or transformations as needed for each query.
+
+ Kylin primarily focuses on **pre-computation** to enhance query performance. However, we also offer advanced features that partially support runtime computation. For more details, please refer to [Table Snapshot](snapshot/snapshot.md), [Runtime Join](features/runtime_join.md), and [Internal Table](../internaltable/intro.md).
+
+
++ **Manual Modeling vs. Recommendation**
+
+ Before Kylin 5.0, model design had to be done manually, which was a tedious process requiring extensive knowledge of multidimensional modeling. However, this changed with the introduction of Kylin 5.0. We now offer a new approach to model design, called **recommendation**, which allows models to be created by importing SQL, along with an automatic way to remove unnecessary indexes. Additionally, the system can leverage query history to generate index recommendations, further optimizing [...]
+
+
++ **Batch Data vs. Streaming Data**
+
+ In the OLAP field, data has traditionally been processed in batches. However, this is changing as more companies are now required to handle both batch and streaming data to meet their business objectives. The ability to process data in real-time has become increasingly critical for applications such as real-time analytics, monitoring, and event-driven decision-making.
+
+ To address these evolving needs, we have introduced support for streaming data in the new version. This allows users to efficiently process and analyze data as it is generated, complementing the traditional batch processing capabilities. For more details, please refer to [Streaming](streaming/intro.md).
diff --git a/website/docs/model/manage/configuration.md b/website/docs/model/manage/configuration.md
new file mode 100644
index 0000000000..81d8eb64bf
--- /dev/null
+++ b/website/docs/model/manage/configuration.md
@@ -0,0 +1,17 @@
+---
+title: Configuration
+language: en
+sidebar_label: Configuration
+pagination_label: Configuration
+toc_min_heading_level: 2
+toc_max_heading_level: 6
+pagination_prev: null
+pagination_next: null
+keywords:
+ - computed column
+draft: false
+last_update:
+ date: 09/13/2024
+---
+
+
diff --git a/website/docs/model/manage/data_loading.md b/website/docs/model/manage/data_loading.md
new file mode 100644
index 0000000000..5d2b77b536
--- /dev/null
+++ b/website/docs/model/manage/data_loading.md
@@ -0,0 +1,144 @@
+---
+title: Data Loading
+language: en
+sidebar_label: Data Loading
+pagination_label: Data Loading
+toc_min_heading_level: 2
+toc_max_heading_level: 6
+pagination_prev: null
+pagination_next: null
+keywords:
+ - data loading
+draft: false
+last_update:
+ date: 08/19/2022
+---
+
+Load data calculates the source data based on the model and index definition. This chapter takes the sample data as an example to introduce two data load methods and processes:
+
+- full load
+- incremental load by date/time
+
+This chapter also introduces segment operations and settings, which are used to manage Segments.
+
+- Segment operation and settings
+ - Segment Merge
+
+### Full Load
+If you want to load all the data in the source table, you can choose full load. The main contents are as follows:
+
+#### <span id="expert">Full Load</span>
+
+If you do not set a time partition column for your model, it will be full load each time.
+
+You cannot merge segments in a full load type model since there should be only one segment.
+
+Here we will introduce how to do a full load in the Web UI:
+
+1. Select the model that needs the full load in the model list. Click the **Build Index** button.
+
+ 
+
+2. You will be prompted to load all data, including the data already loaded in the model.
+
+ 
+
+ > **Note**: If you load data for a model for the first time, the storage size in the above prompt will be 0.00 KB because the model has not been loaded (there is no data in the model).
+
+3. After that, you can view the build index job via the **Monitor -> Job** page.
+
+4. When the data is loaded, you can view the details in the model list. There is only one Segment in the **Segment** tag, and it is marked as full load.
+
+ 
+
+
+### Incremental Load
+
+As your business data grows over time, you can choose to load data by date / time. The main contents are as follows:
+
+#### <span id="expert">Load by Date/Time</span>
+
+If you have set a time partition column for your model, you can load data by date / time each time on the **Data Asset -> Model** page.
+
+1. **First Load**
+ Open Kylin web UI and access the project. Select the model that needs to load in the model list page.
+
+ **Step 1:** Click the **Build Index** button.
+
+ 
+
+ **Step 2:** Select the load range in the pop-up window and click the **Incremental Load** button. This action will trigger the job of **Load Data**.
+
+ > **Note**:
+ >
+ > - You can click the icon on the right side of the load range to automatically detect the latest available range. When your mouse is hovering over the Icon **Detect available range** is displayed.
+ > - When you load historical data at the first time and the data volume is too large, it may lead to a long loading time. Please be sure to set the load range according to your data volume, model complexity, and available resources.
+
+ 
+
+
+
+#### Known limitations
+The start time range of Segment should greater than `1970-01-02`.
+
+
+**Step 3:** After submission, go to the **Monitor -> Job** page, a list of running jobs will be displayed. The first job is the job we just submitted and **Data Range** is displayed as the selected load range in step 2.
+
+**Step 4:** When all steps are complete, the status of the job will become **Finished**. You can view the details in the model list. There is a Segment in the **Segment** tag and it is marked the **Start Time** and **End Time**.
+
+
+
+2. **Incremental Load**
+
+ After the first segment is built, we can build more segments incrementally to accommodate new data. The time range of two segments cannot overlap.
+
+ The steps for incremental load are consistent with the steps described above. Click the **Build Index** button then select the load range in the pop-up window. To ensure continuity, a new segment always starts from the end of the last segment.
+
+ When load completes, go to the model detail page and check there are two segments under the model.
+
+ 
+
+3. **Add Segment**
+
+ Besides the above methods, you can also increase the model data range by adding a new segment. Click the **+ Segment** button under segment list. Then click the **Incremental Build** button in the pop-up window. After that, a building job will be submitted and the third segment will be shown when the job is finished.
+
+ Note: If you want to use this function, you need to enable **Creating Reserved Segments**. Please refer to [Project Settings](../../operations/project-managing/project_settings.md) for more information.
+
+ 
+ 
+
+
+
+### Build Index
+
+As the business scenario changes, some of the indexes in the model need to be retained only in latest months for saving building and storage costs. Therefore, Kylin provides a more flexible way to build indexes since the 5.0 released.
+
+
+#### Build Index
+
+In the **Index Overview** tab, we can see the index list and some basic information. In the **Index List**, we can filter some indexes by the keyword or ids and then only build them in selected segments. For example, some new columns are added in the source table because of the business demands. Therefore, we need to add some new indexes for those columns in the latest one month for analysis and cost saving. As shown in the figure below, we can select all the new and NO BUILD indexes, an [...]
+
+
+
+After that, please select the segment with the latest month and click the Build Index button to generate the building job. If you want to build the segments concurrently to improve the efficiency, you can also check the **Generate multiple segments in parallel** box. Then, the system will generate multiple jobs according to the number of selected segments.
+
+
+
+#### Delete Index
+
+Similar to the building index, you can also delete some indexes in selected segments. For example, deleting some low frequent usage indexes in last year. As shown below, we can choose some of the indexes and click the **Delete** button to choose delete from all segments or parts of them.
+
+Note: If the indexes are deleted from segments, it may influence the query performance because some of query may route to the pushdown engine due to the lack of index.
+
+
+
+#### Build All Index
+
+To support more flexible index building, it may expect that different indexes will be included in different segments. In order to ensure the stable query performance, we recommend you build all index among all segments after a period of time. Therefore, if the index is incomplete, we can quickly build all indexes by clicking the icon after the index data range.
+
+
+
+As shown below, all the segments with incomplete indexes will be shown after clicked the icon. Then, you can select all the segments and click **Build Index** to ensure segments with all indexes.
+
+
+
diff --git a/website/docs/modeling/load_data/images/by_date/buildcube_0.png b/website/docs/model/manage/images/data_loading/by_date/buildcube_0.png
similarity index 100%
rename from website/docs/modeling/load_data/images/by_date/buildcube_0.png
rename to website/docs/model/manage/images/data_loading/by_date/buildcube_0.png
diff --git a/website/docs/modeling/load_data/images/by_date/buildcube_1.png b/website/docs/model/manage/images/data_loading/by_date/buildcube_1.png
similarity index 100%
rename from website/docs/modeling/load_data/images/by_date/buildcube_1.png
rename to website/docs/model/manage/images/data_loading/by_date/buildcube_1.png
diff --git a/website/docs/modeling/load_data/images/by_date/buildcube_2.png b/website/docs/model/manage/images/data_loading/by_date/buildcube_2.png
similarity index 100%
rename from website/docs/modeling/load_data/images/by_date/buildcube_2.png
rename to website/docs/model/manage/images/data_loading/by_date/buildcube_2.png
diff --git a/website/docs/modeling/load_data/images/by_date/buildcube_3.png b/website/docs/model/manage/images/data_loading/by_date/buildcube_3.png
similarity index 100%
rename from website/docs/modeling/load_data/images/by_date/buildcube_3.png
rename to website/docs/model/manage/images/data_loading/by_date/buildcube_3.png
diff --git a/website/docs/modeling/load_data/images/full_load/full_load.png b/website/docs/model/manage/images/data_loading/full_load/full_load.png
similarity index 100%
rename from website/docs/modeling/load_data/images/full_load/full_load.png
rename to website/docs/model/manage/images/data_loading/full_load/full_load.png
diff --git a/website/docs/modeling/load_data/images/full_load/load_data.png b/website/docs/model/manage/images/data_loading/full_load/load_data.png
similarity index 100%
rename from website/docs/modeling/load_data/images/full_load/load_data.png
rename to website/docs/model/manage/images/data_loading/full_load/load_data.png
diff --git a/website/docs/modeling/load_data/images/full_load/notice.png b/website/docs/model/manage/images/data_loading/full_load/notice.png
similarity index 100%
rename from website/docs/modeling/load_data/images/full_load/notice.png
rename to website/docs/model/manage/images/data_loading/full_load/notice.png
diff --git a/website/docs/modeling/load_data/images/load_by_date/add_segment.png b/website/docs/model/manage/images/data_loading/load_by_date/add_segment.png
similarity index 100%
rename from website/docs/modeling/load_data/images/load_by_date/add_segment.png
rename to website/docs/model/manage/images/data_loading/load_by_date/add_segment.png
diff --git a/website/docs/modeling/load_data/images/load_by_date/build_all_index.png b/website/docs/model/manage/images/data_loading/load_by_date/build_all_index.png
similarity index 100%
rename from website/docs/modeling/load_data/images/load_by_date/build_all_index.png
rename to website/docs/model/manage/images/data_loading/load_by_date/build_all_index.png
diff --git a/website/docs/modeling/load_data/images/load_by_date/build_all_index2.png b/website/docs/model/manage/images/data_loading/load_by_date/build_all_index2.png
similarity index 100%
rename from website/docs/modeling/load_data/images/load_by_date/build_all_index2.png
rename to website/docs/model/manage/images/data_loading/load_by_date/build_all_index2.png
diff --git a/website/docs/modeling/load_data/images/load_by_date/build_index.png b/website/docs/model/manage/images/data_loading/load_by_date/build_index.png
similarity index 100%
rename from website/docs/modeling/load_data/images/load_by_date/build_index.png
rename to website/docs/model/manage/images/data_loading/load_by_date/build_index.png
diff --git a/website/docs/modeling/load_data/images/load_by_date/build_index_by_segment.png b/website/docs/model/manage/images/data_loading/load_by_date/build_index_by_segment.png
similarity index 100%
rename from website/docs/modeling/load_data/images/load_by_date/build_index_by_segment.png
rename to website/docs/model/manage/images/data_loading/load_by_date/build_index_by_segment.png
diff --git a/website/docs/modeling/load_data/images/load_by_date/delete_index.png b/website/docs/model/manage/images/data_loading/load_by_date/delete_index.png
similarity index 100%
rename from website/docs/modeling/load_data/images/load_by_date/delete_index.png
rename to website/docs/model/manage/images/data_loading/load_by_date/delete_index.png
diff --git a/website/docs/modeling/load_data/images/load_by_date/load_data.png b/website/docs/model/manage/images/data_loading/load_by_date/load_data.png
similarity index 100%
rename from website/docs/modeling/load_data/images/load_by_date/load_data.png
rename to website/docs/model/manage/images/data_loading/load_by_date/load_data.png
diff --git a/website/docs/modeling/load_data/images/load_by_date/load_ok.png b/website/docs/model/manage/images/data_loading/load_by_date/load_ok.png
similarity index 100%
rename from website/docs/modeling/load_data/images/load_by_date/load_ok.png
rename to website/docs/model/manage/images/data_loading/load_by_date/load_ok.png
diff --git a/website/docs/modeling/load_data/images/load_by_date/load_twice.png b/website/docs/model/manage/images/data_loading/load_by_date/load_twice.png
similarity index 100%
rename from website/docs/modeling/load_data/images/load_by_date/load_twice.png
rename to website/docs/model/manage/images/data_loading/load_by_date/load_twice.png
diff --git a/website/docs/modeling/load_data/images/load_by_date/model_list_3.png b/website/docs/model/manage/images/data_loading/load_by_date/model_list_3.png
similarity index 100%
rename from website/docs/modeling/load_data/images/load_by_date/model_list_3.png
rename to website/docs/model/manage/images/data_loading/load_by_date/model_list_3.png
diff --git a/website/docs/modeling/load_data/images/load_by_date/notice_2.png b/website/docs/model/manage/images/data_loading/load_by_date/notice_2.png
similarity index 100%
rename from website/docs/modeling/load_data/images/load_by_date/notice_2.png
rename to website/docs/model/manage/images/data_loading/load_by_date/notice_2.png
diff --git a/website/docs/modeling/images/model_overview/dimensions.png b/website/docs/model/manage/images/dimensions.png
similarity index 100%
rename from website/docs/modeling/images/model_overview/dimensions.png
rename to website/docs/model/manage/images/dimensions.png
diff --git a/website/docs/modeling/images/model_overview/er.png b/website/docs/model/manage/images/er.png
similarity index 100%
rename from website/docs/modeling/images/model_overview/er.png
rename to website/docs/model/manage/images/er.png
diff --git a/website/docs/modeling/model_design/advance_guide/images/model_check.png b/website/docs/model/manage/images/import_export/model_check.png
similarity index 100%
rename from website/docs/modeling/model_design/advance_guide/images/model_check.png
rename to website/docs/model/manage/images/import_export/model_check.png
diff --git a/website/docs/modeling/model_design/advance_guide/images/model_export.png b/website/docs/model/manage/images/import_export/model_export.png
similarity index 100%
rename from website/docs/modeling/model_design/advance_guide/images/model_export.png
rename to website/docs/model/manage/images/import_export/model_export.png
diff --git a/website/docs/modeling/model_design/advance_guide/images/model_upload.png b/website/docs/model/manage/images/import_export/model_upload.png
similarity index 100%
rename from website/docs/modeling/model_design/advance_guide/images/model_upload.png
rename to website/docs/model/manage/images/import_export/model_upload.png
diff --git a/website/docs/modeling/images/model_overview/measures.png b/website/docs/model/manage/images/measures.png
similarity index 100%
rename from website/docs/modeling/images/model_overview/measures.png
rename to website/docs/model/manage/images/measures.png
diff --git a/website/docs/modeling/images/model_list.png b/website/docs/model/manage/images/model_list.png
similarity index 100%
rename from website/docs/modeling/images/model_list.png
rename to website/docs/model/manage/images/model_list.png
diff --git a/website/docs/modeling/images/modellist_more_info.png b/website/docs/model/manage/images/modellist_more_info.png
similarity index 100%
rename from website/docs/modeling/images/modellist_more_info.png
rename to website/docs/model/manage/images/modellist_more_info.png
diff --git a/website/docs/modeling/model_design/advance_guide/images/model_publish.png b/website/docs/model/manage/images/publish/model_publish.png
similarity index 100%
rename from website/docs/modeling/model_design/advance_guide/images/model_publish.png
rename to website/docs/model/manage/images/publish/model_publish.png
diff --git a/website/docs/modeling/load_data/segment_operation_settings/images/segment.png b/website/docs/model/manage/images/segment/segment.png
similarity index 100%
rename from website/docs/modeling/load_data/segment_operation_settings/images/segment.png
rename to website/docs/model/manage/images/segment/segment.png
diff --git a/website/docs/modeling/images/model_overview/unfold_model.png b/website/docs/model/manage/images/unfold_model.png
similarity index 100%
rename from website/docs/modeling/images/model_overview/unfold_model.png
rename to website/docs/model/manage/images/unfold_model.png
diff --git a/website/docs/modeling/model_design/advance_guide/model_metadata_managment.md b/website/docs/model/manage/import_export.md
similarity index 95%
rename from website/docs/modeling/model_design/advance_guide/model_metadata_managment.md
rename to website/docs/model/manage/import_export.md
index a634c924b2..7955235098 100644
--- a/website/docs/modeling/model_design/advance_guide/model_metadata_managment.md
+++ b/website/docs/model/manage/import_export.md
@@ -1,8 +1,8 @@
---
-title: Model Metadata Management
+title: Import and Export
language: en
-sidebar_label: Model Metadata Management
-pagination_label: Model Metadata Management
+sidebar_label: Import and Export
+pagination_label: Import and Export
toc_min_heading_level: 2
toc_max_heading_level: 6
pagination_prev: null
@@ -24,7 +24,7 @@ The movement of models in different environments is an important process of actu
In many companies, in order to ensure the stability of the production environment, the release and change of models in the production environment is very strict. Users often need to deploy an additional independent development environment for data development and test verification (and possibly there is still a test environment), and migrating the data model from the development environment to the production environment is the process of model release. At the same time, this process need [...]
-
+
### Model Metadata Export {#model_metadata_export}
@@ -57,7 +57,7 @@ Notes:
- You can click the **Export Model** button on the **Model List Page**, select multiple models and export.
- Or click the **Admin** button on the right of the status bar at the top of the page, in the project list page, in a single project **Actions - More Actions - Export Model**, select multiple models and export.
-
+
### Model Metadata Import {#model_metadata_import}
@@ -68,7 +68,7 @@ Notes:
- Or enter the **Admin** page, on the project list page, select the project to import the model metadata, in **Action - ...(More Actions) - Import Model**, upload the model metadata compression package.
- 
+ 
#### Select operation type
@@ -109,7 +109,7 @@ When the model to be imported is selected, the model to be imported will be disp
After the model is imported, it may be necessary to build the newly added index before it can serve queries.
-
+
### Some Practice {#model_io_practice}
@@ -123,7 +123,7 @@ Different environments generally have different values of multi-level partition
#### Model metadata export and import related API
-You can also export and import model metadata through API. For details, please refer to: [Model Import and Export API](../../../restapi/model_api/model_import_and_export_api.md)
+You can also export and import model metadata through API. For details, please refer to: [Model Import and Export API](../../restapi/model_api/model_import_and_export_api.md)
### Known limitation {#know_limitation}
diff --git a/website/docs/model/manage/model_info.md b/website/docs/model/manage/model_info.md
new file mode 100644
index 0000000000..753ae10275
--- /dev/null
+++ b/website/docs/model/manage/model_info.md
@@ -0,0 +1,87 @@
+---
+title: Model Information
+language: en
+sidebar_label: Model Information
+pagination_label: Model Information
+toc_min_heading_level: 2
+toc_max_heading_level: 6
+pagination_prev: null
+pagination_next: null
+keywords:
+ - model management
+ - model operations
+draft: false
+last_update:
+ date: 09/13/2024
+---
+
+
+### Model List
+
+ You can create and design models manually. Below are the main content of model list:
+
+1. Log in to Web UI, switch to a project.
+
+2. Navigate to **Data Asset -> Model** page, where models are shown in a list. The picture below is the index group list:
+
+ 
+
+ **Fields Explanation:**
+
+ - **Model Name**: Model's name.
+
+ - **Status**: There are four statuses.
+ - *ONLINE* indicates this model is online and is able to answer queries.
+ - *OFFLINE* indicates this model is offline and not available to answer queries. We recommend using offline when you need to edit the model.
+ - *BROKEN* indicates this model is broken and not available. Mostly happens when the schemas of related source tables have changed, for instance, a related source table is deleted.
+ - *WARNING* indicates this model is warning and can only server parts of queries. Mostly happens when the segments exist holes or indexes are waiting to build.
+ - **Last Updated Time**: The lastest time to update model.
+
+ - **More Actions**: The **More Actions** button will appear when you are hovering on model name area, please refer to [Model Operations](#operation) for details.
+
+ - **Owner**: The user who created this model.
+
+ - **Description**: Model description.
+
+ - **Fact Table**: The fact table of this model.
+
+ - **Types**: Model types, which include *Batch Model*
+
+ - **Usage**: Hit count by SQL statements in the last 30 days. Update every 30 minutes.
+
+ - **Rows**: The rows of loaded data in this model.
+
+ - **Storage**: The storage size of loaded data in this model, which combines the storage size of all Segments data.
+
+ > Tip: When the tiered storage is turned on, the total storage size of the data loaded into the tiered storage (ClickHouse) will be displayed.
+
+ - **Expansion Rate**: The ratio of the storage size of the built data to the storage size of the corresponding source table data under the model. Expansion Rate = Storage Size / Source Table Size.
+
+
+ > Notice: The expansion rate won't show if the storage size is less than 1GB.
+
+ - **Index Amount**: The amount of indexes in this model.
+
+### Model view
+
+After expanding the model information, you can see the model overview page, which will help you to quickly get the model information.
+
+
+
+On this page, you can view the ER diagram of the model.
+
+
+
+What's more, you can view the dimensions and measures information contained in the model.
+
+
+
+
+
+
+### Model Details
+
+Models contain Segments and indexes. You can click model name to unfold the detailed information, as shown below:
+
+
+
diff --git a/website/docs/model/manage/model_operation.md b/website/docs/model/manage/model_operation.md
new file mode 100644
index 0000000000..fa9338a5cb
--- /dev/null
+++ b/website/docs/model/manage/model_operation.md
@@ -0,0 +1,53 @@
+---
+title: Model Operation
+language: en
+sidebar_label: Model Operation
+pagination_label: Model Operation
+toc_min_heading_level: 2
+toc_max_heading_level: 6
+pagination_prev: null
+pagination_next: null
+keywords:
+ - model management
+ - model operations
+draft: false
+last_update:
+ date: 09/13/2024
+---
+
+
+
+
+### <span id="operation">Model Operation</span>
+
+You are only allowed to operate on models. You can hover on the right most column **Actions** of the model list to get the action names. Specific actions are listed below:
+
+- **Edit**: Click on the pencil shape button, enter into the model editing page.
+
+- **Build Index**: Loads data for models. You can choose the data range in the pop-up window.
+
+- **Model Partition**: Set partition column for the model.
+
+- **Export Model**: Export single model metadata.
+
+ > **Note**: Since the locked indexes will be deleted after the new indexes have been built, the exported model metadata will not include the locked index.
+
+- **Export TDS**: Export TDS file of the model .
+
+- **Rename**: Renames the model.
+
+- **Clone**: Clones an identical model. You can give a new name for this new model. The new model has the same fact table, dimension tables, join relationship, dimensions, measures, computed columns, date partition column, aggregate indexes, table indexes, etc. as the origin model. But the new model does not have data, you need to load data for this cloned model manually.
+
+ > **Note**: Since the locked indexes will be deleted after the new indexes have been built, the cloned model will not include the locked index.
+
+- **Change Owner**：Change model owner. Only system administrators and project administrators have the authority to modify model owner.
+
+- **Delete**: Deletes the model, remove the loaded data at the same time.
+
+- **Purge**: Purges all loaded data in this model.
+
+- **Offline**: Makes a *Online / Warning* model offline. An offline model cannot answer any queries.
+
+- **Online**: Makes a *Offline* model online. An online model should be able to answer related queries.
+
+> **Note:** If the model is in *BROKEN* status, only the **delete** operation is allowed.
diff --git a/website/docs/modeling/load_data/segment_operation_settings/segment_merge.md b/website/docs/model/manage/segment.md
similarity index 67%
rename from website/docs/modeling/load_data/segment_operation_settings/segment_merge.md
rename to website/docs/model/manage/segment.md
index 763df9ecda..a8a0caa723 100644
--- a/website/docs/modeling/load_data/segment_operation_settings/segment_merge.md
+++ b/website/docs/model/manage/segment.md
@@ -1,8 +1,8 @@
---
-title: Segment Merge
+title: Segment
language: en
-sidebar_label: Segment Merge
-pagination_label: Segment Merge
+sidebar_label: Segment
+pagination_label: Segment
toc_min_heading_level: 2
toc_max_heading_level: 6
pagination_prev: null
@@ -14,6 +14,86 @@ last_update:
date: 08/19/2022
---
+Model (index group) consists of one or more segments. Each segment contains a range of data. Segment is created by building index or loading data with a selected data range on the partition columns.
+
+The main contents are as follows:
+
+
+### <span id="view">View Segment</span>
+
+User can access the Segment management interface by following these steps:
+
+1. Open **Data Asset -> Model** page, and click the model (index group) name.
+2. Select **Segment** tag.
+
+The model list page in the project is shown as below.
+
+
+
+Field description of the Segment list:
+
+- Start time: The start time of the data in the Segment. If it is loaded in full **"Full Load"** is displayed.
+
+- End time: The end time of the data in the Segment. If it is loaded in full **"Full Load"** is displayed.
+
+- Index: Indexes in this segment / Total indexes
+
+- Status: Segment status. You can find a detailed introduction [Segment Status](#status) section.
+
+- Last Updated Time: Segment last updated time.
+
+- Source Records: The source records of the data in the segment.
+
+- Storage: The storage size of the data in the segment.
+
+ > Tip: When the tiered storage is turned on, the storage size of the data loaded into the tiered storage (ClickHouse) will be displayed.
+
+- Actions: The operation of the segment. Currently only **Show Detail** is supported.
+
+
+
+### <span id="status">Segment Status</span>
+
+You can view the segment status in the segment list. There are 6 types of segment statuses. See below:
+
+- **ONLINE**: Segment can serve the query by indexes loaded data or pushdown engine.
+- **WARNING**: The data in the segment has been loaded and can serve the query. However, the source data might be changed which might cause the data inconsistent. It's highly recommended to refresh all indexes within this segment.
+- **LOCKED**: Segments that are refreshing or merging will be locked.
+- **LOADING**: The data in the segment is loading.
+- **REFRESHING**: A new segment is automatically generated when you refresh the specified segment. This new segment is marked as *REFRESHING*. When the refresh is complete the old segment will be automatically deleted.
+- **MERGING**: A new segment is automatically generated when you merge the specified segments. This new segment is marked as *MERGING*. When the merge is complete the old segment will be automatically deleted.
+
+
+
+### <span id="expert_operation">Segment Operation</span>
+
+You have 6 types of segment operations on the **Data Asset -> model **page.
+
+- **+ Segment**: Add segments to define the model’s data range for serving queries. Queries within the range could be answered by indexes or pushdown engine. Queries out of the range would have no results. The button is located above the segment list.
+
+> **Note:** In **Setting -> Basic Settings -> Segment Settings**, enable **Creating Reserved Segments**, then the **+ Segment** operation button will appear.
+
+- **Show Detail**: You can click the icon on the right side of the segment list. When your mouse is hovering over the icon **Show Detail** is displayed. You can view details such as storage size, the data range and more.
+
+- **Refresh**: Refresh the data in the segment. This operation supports batch refresh. The **Refresh** button is located above the segment list.
+
+ >**Note**: Only ONLINE and WARNING status segments can be refreshed.
+
+- **Merge**：Merge multiple segments as a new one. The **Merge** button is located above the segment list.
+
+ > **Note**: Only ONLINE and WARNING status segments can be merged.
+
+- **Delete**: Delete the segment. This operation supports batch deletion. The **Delete** button is located above the segment list.
+
+- **Fix**: Fix the discontinuous segments. This button will be only displayed above the Segment list when the holes exists in Segment ranges.
+
+
+
+### <span id="setting"> Segment Settings</span>
+
+You can set some to manage segments automatically in the **Setting -> Segment Settings** page. Please refer to [Project Settings](../../operations/project-managing/project_settings.md) for the specific requirements.
+
+
In the incremental build mode, as the number of segments increases, the system may need to aggregate multiple segments to serve the query, which degrades the query performance and the query performance decreases. At the same time, a large number of small files will put pressure on the HDFS Namenode and affect the HDFS performance. Apache Kylin provides a mechanism to control the number of segments - **Segments Merge **.
@@ -41,7 +121,7 @@ According to different business needs, it supports the automatic merging of proj
- Project-level: Used for all models in a project, with the same merge strategy.
- Model-level: used for multiple models in a project, with different automatic merging strategies.
-Please refer to **Segment Settings** and **Model/Index Group Rewrite Settings** of [Project Settings](../../../operations/project-managing/project_settings.md) for the specific requirements.
+Please refer to **Segment Settings** and **Model/Index Group Rewrite Settings** of [Project Settings](../../operations/project-managing/project_settings.md) for the specific requirements.
#### <span id="strategy">Auto-merge strategy</span>
- Merge Timing: The system triggers an automatic merge attempt every time a new segment in the project becomes complete. To ensure query performance, all segments will not be merged at once.
diff --git a/website/docs/model/manual/agg_group_example.md b/website/docs/model/manual/agg_group_example.md
new file mode 100644
index 0000000000..e5136a839d
--- /dev/null
+++ b/website/docs/model/manual/agg_group_example.md
@@ -0,0 +1,156 @@
+
+
+[//]: # (Need move to blog)
+
+### Examples
+
+In this section, we will present several practical examples to guide you through the process of designing aggregate groups effectively. These examples will illustrate different scenarios and considerations, helping you understand how to tailor your aggregation groups to meet specific analytical needs and optimize query performance. By following these examples, you can gain insights into best practices and strategies for creating efficient and effective aggregate groups in Kylin.
+
+#### Aggregate Group
+
+Assuming a transactional aggregate index that includes the following dimensions: **Customer ID** (*buyer_id*), **Transaction Date** (*cal_dt*), **Payment Type** (*pay_type*), and **Customer City** (*city*).
+
+Analysts may need to analyze the data in different ways depending on their objectives. For instance, there are instances when they require grouping the dimensions *city*, *cal_dt*, and *pay_type* to gain insights into the various payment types used in different cities. At other times, they might want to group *city*, *cal_dt*, and *buyer_id* to understand customer behavior across different locations.
+
+To accommodate these analytical needs, we recommend creating two separate aggregate groups:
+
+1. **Aggregate-Group-1**:
+ - **Dimensions**: *city*, *cal_dt*, *pay_type*
+ - **Purpose**: To analyze the impact of payment types across different cities over time.
+
+2. **Aggregate-Group-2**:
+ - **Dimensions**: *city*, *cal_dt*, *buyer_id*
+ - **Purpose**: To evaluate customer behavior by examining transactions in various cities over time.
+
+
+
+
+Regardless of other situations, those two aggregate groups can reduce 3 unnecessary index: [pay_type, buyer_id]、[city,pay_type, buyer_id] and [cal_dt, pay_type, buyer_id], so storage space and build time can be saved.
+
+```sql
+-- will hit on index [cal_dt, city, pay_type]
+Select cal_dt, city, pay_type, count(*) from table
+Group by cal_dt, city, pay_type;
+
+-- will hit on index [cal_dt, city, buyer_id]
+Select cal_dt, city, buyer_id, count(*) from table
+Group by cal_dt, city, buyer_id;
+
+-- No index can be hit, so Kylin will calculate the result based on existing index on-demand
+Select pay_type, buyer_id,count(*) from table
+Group by pay_type, buyer_id.
+```
+
+#### Mandatory Dimension
+
+Assume a transactional aggregate index that includes transaction date, transaction location, product, payment type. Transaction date is a frequently used group by dimension. If transaction date is set as a mandatory dimension, the combination of dimensions will be calculated as shown below:
+
+
+
+#### Hierarchy Dimension
+
+Assume a transactional aggregate index that includes dimensions transaction city `city`, transaction province `province`, transaction country `country` and payment type `pay_type`. Analysts will group transaction country, transaction province, transaction city, and payment type together to understand customer payment type preference in different geographical locations. In the example above, we recommend creating hierarchy dimensions in existing aggregate group (Country / Province / City) [...]
+
+
+
+Aggregate Group: `[country, province, city，pay_type]`
+
+Hierarchy Dimension: `[country, province, city]`
+
+
+Case 1:
+
+Analysts want to understand city level customer payment type preferences
+
+```sql
+SELECT city, pay_type, count() FROM table GROUP BY city, pay_type
+```
+
+can be retrieved from index [country, province, city, pay_type].
+
+
+
+Case 2:
+
+Analysts want to understand province level customer payment type preferences
+
+```sql
+SELECT province, pay_type, count() FROM table GROUP BY province, pay_type
+```
+
+can be retrieved from index [country, province, pay_type].
+
+
+
+Case 3:
+
+Analysts want to understand customer's payment type preferences from country level
+
+```sql
+SELECT country, pay_type, count() FROM table GROUP BY country, pay_type
+```
+
+can be retrieved from index [country, pay_type].
+
+
+
+Case 4:
+
+Analysts want to reach a different granularity level using the geographical dimension:
+
+```sql
+SELECT country, city, count(*) FROM table GROUP BY country, city
+```
+
+
+will retrieve data from index [country, province, city].
+
+#### Joint Dimension
+
+Assume a transactional aggregate index that includes dimension transaction date `cal_dt`, transaction city `city`, customer gender `sex_id`, payment type `pay_type`. Analysts usually need to group transaction date, transaction city, and customer gender to understand consumption preference for different genders in different cities, in this case, `cal_dt, city, sex_id `will be grouped together. In the case above, we recommend assigning them to joint dimensions based on existing aggregate g [...]
+
+
+
+Aggregate Group: `[cal_dt,city, sex_id，pay_type]`
+
+Joint Dimension: `[cal_dt, city, sex_id]`
+
+
+
+Case 1:
+
+```sql
+SELECT cal_dt,city, sex_id, count(*) FROM table GROUP BY cal_dt, city, sex_id
+```
+
+can retrieve data from index [cal_dt, city, sex_id].
+
+Case 2:
+
+```sql
+SELECT cal_dt, city, count(*) FROM table GROUP BY cal_dt, city
+```
+
+then no index can be hit, Kylin will leave calculate result based on existing index.
+
+
+
+#### ShardBy Column
+
+```
+select org_id, cust_id, sum1, sum2
+from (
+ select org_id, cust_id, sum(....) sum1
+ from fact
+ where dt = ...
+ group by org_id, cust_id
+) T1
+inner join (
+ select cust_id, sum(...) sum2
+ from fact
+ where dt = ...
+ group by cust_id
+) T2 on T1.cust_id = T2.cust_id
+```
+
+The above SQL can be queried by joining two aggregate indexes. To optimize the join operation, user can set the join key - `cust_id` as the ShardBy column. In that case the shuffle stage can be skipped for the join operation.
diff --git a/website/docs/model/manual/aggregation_group.md b/website/docs/model/manual/aggregation_group.md
new file mode 100644
index 0000000000..fa560d627c
--- /dev/null
+++ b/website/docs/model/manual/aggregation_group.md
@@ -0,0 +1,255 @@
+---
+title: Aggregate Index
+language: en
+sidebar_label: Aggregate Index
+pagination_label: Aggregate Index
+toc_min_heading_level: 2
+toc_max_heading_level: 6
+pagination_prev: null
+pagination_next: null
+keywords:
+ - aggregate index
+draft: false
+last_update:
+ date: 09/13/2024
+---
+
+Kylin is renowned for enhancing query performance by pre-calculated aggregate groups, which represent various combinations of all dimensions, also known as cuboids. However, it faces the well-known challenge of the **curse of dimensionality**—as the number of dimensions increases, the number of possible indices grows exponentially (2^n - 1). For example, with 3 dimensions, there are 7 possible indices, but with 10 dimensions, the number of cuboids surges to 1023.
+
+To address this challenge, Kylin employs an **aggregation group mechanism** that optimizes the management of high-dimensional data, helping to reduce the complexity and significantly improve query performance.
+
+
+### <span id="background">Basic Concepts</span>
+
+To create aggregate indexes, it is important to understand the following key concepts:
+
+- **Aggregate Group**: Defines specific sets of dimensions that should be aggregated together, limiting the scope of combinations to what is most relevant for queries.
+- **Joint Dimension**: Groups dimensions that are frequently queried together, helping to optimize index creation for common query patterns.
+- **Hierarchy Dimension**: Establishes a hierarchy among dimensions, ensuring that when one dimension is queried, all related dimensions are considered, reducing unnecessary combinations.
+- **Mandatory Dimension**: Marks certain dimensions as essential, ensuring they are always included in index combinations, which helps avoid generating irrelevant cuboids.
+
+These settings are crucial for optimizing the index-building process, as they minimize both storage and computation costs while maintaining efficient query performance.
+
+In this chapter, we will first guide you through the process of designing an **Aggregation Group**. Following that, we will delve into the principles behind aggregation groups, equipping you with the knowledge to design them effectively.
+
+The following GIF demonstrates how to navigate to the Aggregation Group page.
+
+
+
+### <span id="edit">Add Aggregate Group</span>
+
+On the **Aggregation Group** page, when you hover over the  button, a tooltip will appear displaying **Add Aggregate Group**. By clicking this button, you will be redirected to the page for adding an aggregate group, as shown below.
+
+
+
+Alternatively, you can navigate to the same page by clicking the **Aggregate Group** button. Both methods will take you to the interface where you can define the dimensions and measures for the aggregation group, and configure advanced settings such as **Mandatory**, **Hierarchy**, and **Joint** dimensions. Additionally, there is an option to set the **Max Dimension Combination**, which limits the maximum number of dimension combinations [...]
+
+#### Setting Dimensions
+
+When you click the **Add** button on the **Include** sub-page, a dialog box labeled **Edit Included Dimensions** will appear. This dialog allows you to select the dimensions for the aggregate group. In this example, we will select all available dimensions and then click the **OK** button to confirm the selection.
+
+
+
+
+After selecting the dimensions to be included, the next step is to configure the **Mandatory Dimensions**, **Hierarchy Dimensions**, and **Joint Dimensions**. Tooltips are available to assist in understanding these settings—when the icon  was clicked, they provide explanatory messages. For instance, the tooltip for **mandatory dimension** shows: "If a dimension is set as a mandatory dimension, only the index with this dimension will be calculated."
+
+In our example:
+
+- *LO_ORDERDATE* is designated as a mandatory dimension, ensuring that any query using the aggregation group must include this dimension.
+- *LO_CUSTKEY* and *LO_SUPPKEY* are configured as joint dimensions, indicating that these dimensions are frequently queried together and should be optimized accordingly.
+- *D_DATE*, *D_MONTH*, and *D_YEAR* are set as hierarchy dimensions, establishing a parent-child relationship that allows for drill-down queries, such as from year to month to date.
+
+
+
+#### Max Dimension Combination
+
+At the bottom of the page, you'll notice a label of *Max Dimension Combination*. This setting applies globally to all aggregation groups within the model, meaning it defines the maximum number of dimension combinations allowed for the entire model. Each individual aggregation group also has its own setting, which works independently for that specific group.
+
+In our example, the max dimension combination for the model is set to 4, meaning the maximum number of dimension combinations across all aggregation groups cannot exceed this limit. However, for the aggregation group *Aggregate-Group-1*, the limit is set to 3, meaning this particular group can only combine up to 3 dimensions.
+
+#### Setting Measures
+
+When you choose the page of **Measure**, you can define all the measures of this aggregate group. By default, all the measures of model are included in the aggregate group, if you don't need all of them, you can edit the content.
+
+
+
+#### Aggregate Group View
+
+
+After completing all the steps mentioned above, you'll have an aggregation group ready to be saved, as shown in the picture above. If you want to add more aggregation groups, simply click the **Add Aggregate Group** button. In this example, we have defined just one aggregation group. Once you're done, click the **Save** button, and you will be directed back to the **Aggregation Group** page as follows.
+
+
+
+#### Text Recognition
+
+You may notice the **Text Recognition** button, which is particularly useful when your model contains a large number of dimensions. This feature allows you to prepare the dimensions in advance, paste them into the provided text box, and then execute the process. Kylin will automatically extract the correct dimensions from the input. In this example, we just manually input the desired dimensions into the text box, separating each dimension with a comma, as shown below.
+
+
+
+#### Index Overview
+
+If you want to know the indexes generated by the aggregate group, you should navigate to the **Index Overview** page. This page gives you all the information of index, we will introduce them in the chapter of [model info](../manage/model_info.md).
+
+
+
+#### Advanced Settings
+
+On the **Aggregate Group** page, you’ll notice an icon (). Clicking this icon will navigate you to the **Advanced Setting** page. Here, you can specify columns as **ShardBy** as needed.
+
+The **ShardBy** column is typically one of the filter or grouping dimensions that are frequently used in queries. While you can designate more than one shard by dimensions, only the first one will be applied during queries. Therefore, it’s important to choose this column wisely, balancing factors such as column cardinality. Cardinality information is generated via [table sampling](../../datasource/data_sampling.md).
+
+This setting applies to all aggregate groups. If an aggregate group does not contain the specified shard by dimensions, the system will automatically bypass them when generating aggregate indexes.
+
+
+
+
+
+
+
+Based on your business scenario, you can decide whether to add an index that includes the dimensions and measures defined across all aggregate groups. This index can support queries that span multiple aggregate groups but may not achieve the optimal query performance. Besides, incorporating this index comes with higher storage and building costs. How to set this configuration, please refer to the [configuration](manage/configuration.md) chapter.
+
+```shell
+# generating base cuboid of all aggregate groups, by default this is true
+kylin.cube.aggrgroup.is-base-cuboid-always-valid=true
+```
+
+Please note, this is different from the **Base Index**. The dimensions and measures in the aggregate groups may not cover everything in the model, and the base table index will not be generated in this context.
+
+### Principles
+
+Now that we have completed the design of the aggregation group, we will delve deeper into the underlying principles in this section. Understanding these principles will help you optimize the performance and management of aggregation groups in Kylin, ensuring efficient query processing and resource utilization.
+
+
+#### <span id="agg">Aggregate Group</span>
+
+You can categorize the combinations of dimensions and measures into several groups, referred to as **Aggregate Groups**. For instance, as illustrated at the beginning of this chapter, if you only require the dimension combinations of *[A B M1]* and *[C D M2]*, the aggregate index group can be segmented into two distinct aggregate groups: *A B-M1* and *C D-M2*. This approach effectively reduces the total number of indexes from 15 to 7, optimizing storage and improving query performance.
+
+
+
+
+The aggregate groups required by end users may contain overlapping dimensions. For instance, consider aggregate group ABC and aggregate group BCD, both of which include dimensions B and C. If these two aggregate groups share the same measures, they will generate identical indexes; for example, both aggregate group ABC and aggregate group BCD will derive the index BC. However, Kylin ensures that an index is not generated multiple times. If an index can be derived from more than one aggreg [...]
+
+
+
+#### <span id="mandatory">Mandatory Dimension</span>
+
+When you want to focus your analysis on one or more specific dimensions that must be included in every query, you can designate these dimensions as **Mandatory Dimensions**. This setting ensures that only indexes containing the mandatory dimension will be calculated and considered for queries.
+
+For instance, if we take the aggregate index example from earlier in the chapter, where there were originally 15 possible indexes based on various combinations of dimensions, and we set dimension **A** as mandatory, the result would be a reduced set of indexes. In this case, only those indexes that include dimension **A** will be calculated, effectively limiting the combinations and simplifying the number of indexes from 15 to 8.
+
+
+
+
+
+#### <span id="hierarchy">Hierarchy Dimension</span>
+
+End users typically utilize dimensions with hierarchical relationships, such as country, province, and city. In this context, a hierarchical relationship can be defined as a **Hierarchy Dimension**. From top to bottom, the hierarchy consists of a one-to-many relationship: country, province, and city. These three dimensions can be grouped into the following combinations:
+
+1. Group by country.
+
+2. Group by country and province (equivalent to grouping by province).
+
+3. Group by country, province, and city (equivalent to grouping by country and city, or simply by city).
+
+In the aggregate index group outlined below, let dimension A represent Country, dimension B represent Province, and dimension C represent City. Here, dimensions A, B, and C can be configured as a hierarchical dimension. Consequently, the following index relationships hold true:
+
+- Index [A, C, D] is equivalent to index [A, B, C, D].
+- Index [B, D] is equivalent to index [A, B, D].
+
+Thus, indexes [A, C, D] and [B, D] can be pruned effectively.
+
+
+
+As the diagram below illustrates, based on the method above, Kylin can prune redundant index, hence reducing index from 15 to 7.
+
+
+
+#### <span id="joint">Joint Dimension</span>
+
+Sometimes you don’t need details of all possible combination dimensions. For example, you might query dimension A, B, C together in most cases but not dimension A, C or dimension C alone. To enhance performance in this case, **Join Dimension** can be used. If A, B, and C are defined as Joint Dimension, Kylin will only build index ABC but not index AB, BC and A. Finally, indices will be built as below. The number of indices can then be reduced from 15 to 3.
+
+
+
+**Dimensional cardinality product **
+
+The *Dimensional Cardinality Product* refers to the product of the cardinalities of each dimension field within a joint dimension. This cardinality data is derived from the sampling results of the source data table. The dimensional cardinality product represents the maximum number of possible permutations and combinations of dimensions in the joint index (i.e., the maximum number of records for that index).
+
+Although the dimensional cardinality product does not directly participate in the index creation process, it serves as a useful reference when defining the union dimension of an aggregate group. To ensure optimal query performance, it is generally recommended that the dimensional cardinality product of a union dimension does not exceed 100,000. In special cases, where dimensions in the union dimension must always be queried together, this value can be less of a concern.
+
+#### ShardBy Column
+
+In some cases, subqueries within a join may match two indexes. When this happens, Kylin retrieves data from both indexes and performs the join operation. However, if the cardinality of the subquery is high, the join operation may become time-consuming. To optimize this, users can set the *ShardBy* column to the join key.
+
+```shell
+kylin.storage.columnar.expose-sharding-trait=true
+```
+
+This feature is enabled by default, Kylin will expose the sharding information to Spark. Normally, joining two large datasets in Spark requires a data shuffle beforehand to achieve hash partitioning. However, if the *ShardBy* column has already partitioned the data, the shuffle stage can be skipped, significantly speeding up the join operation.
+
+:::note Limitations
+Since there can only be one **ShardBy** column for a model, the optimization applies exclusively to joins that involve this single **ShardBy** column. For joins on other columns, the optimization will not take effect, and a data shuffle will still be required in Spark. Therefore, it’s important to choose the **ShardBy** column carefully, particularly when frequent joins occur on that specific column.
+:::
+
+
+#### <span id="mdc">Max Dimension Combination</span>
+
+The use of aggregate groups helps prevent index explosion, but achieving optimal index performance requires a solid understanding of the data model, which can be challenging for junior modelers.
+
+This section introduces a simple index pruning tool called **Max Dimension Combination (MDC)**. The MDC specifies the maximum number of dimensions allowed in a single index, preventing the creation of indexes with excessive dimensions during the index-building process. The MDC is especially useful when most queries interact with no more than N dimensions, where N is the configurable MDC threshold. This helps streamline index creation and improve query performance by focusing on the most [...]
+
+The following graph illustrates a scenario with 7 dimensions, with some details hidden for clarity:
+
+- When the MDC is set to 4, indexes with more than 4 dimensions, such as *ABCDEF*, *ABCDEG*, *ABCDE*, and *ABCDF*, will be pruned.
+- When the MDC is set to 3, indexes like *ABCDEF*, *ABCDEG*, *ABCD*, and *ABCE* will be excluded.
+
+
+
+To optimize index building, creating an index that includes dimensions and measures from all aggregate groups ensures it won't be pruned. Remember that joint dimensions and hierarchy dimensions are counted as a single dimension, while mandatory dimensions are ignored. When using the **Max Dimension Combination (MDC)** tool, consider the actual number of dimensions in the index, especially with joint or hierarchy dimensions, to ensure effective pruning aligned with query performance.
+
+The MDC dimension pruning tool significantly reduces the number of indexes and storage size. However, complex queries that involve more dimensions may encounter larger indexes, which can slow down query responses due to online calculations. Like other optimization tools, this involves a trade-off. If most of your queries touch fewer dimensions, using MDC is beneficial.
+
+Laboratory tests have shown that when the number of dimensions in the aggregate group is large, checking the index count can take several minutes, potentially causing web UI lags. Here are some test results for reference:
+
+- **1 aggregate group with 1000 dimensions**: MDC set to 1, average check time is **3.1 minutes**.
+- **1 aggregate group with 1500 dimensions**: MDC set to 1, average check time is **13.9 minutes**.
+- **3 aggregate groups with 500 dimensions each**: MDC set to 1, average check time is **3 minutes**.
+
+Please consider these results as general guidance and assess based on your specific scenario.
+
+#### How to Count Dimensions
+
+Next, let’s discuss how to count the number of dimensions in a query. This count refers to the dimensions represented in the corresponding index.
+
+- For ordinary dimensions, each counts as one dimension.
+- A group of joint dimensions or hierarchy dimensions is treated as a single dimension.
+- Mandatory dimensions are ignored.
+
+For example, in the following SQL query:
+
+```sql
+SELECT
+ COUNT(*)
+FROM
+ table
+GROUP BY
+ column_mandatory,
+ column_joint1,
+ column_joint2,
+ column_hierarchy1,
+ column_hierarchy2,
+ column_normal;
+```
+
+We can analyze the dimensions as follows:
+
+- **1** mandatory dimension.
+- **2** dimensions from a single joint dimension.
+- **2** dimensions from a single hierarchy dimension.
+- **1** normal dimension.
+
+Thus, for index pruning, this query is considered to involve **three dimensions**. This counting method helps ensure that the index pruning mechanism effectively limits the number of dimensions in the created indexes.
+
+### Summary
+
+In this chapter, we covered how to create aggregate groups and the principles behind them. However, designing effective indexes seems quite complex, especially for beginners. If you're looking for a simpler approach, please refer to the [recommendation](../rec/intro.md) for more details.
+
diff --git a/website/docs/model/manual/computed_column.md b/website/docs/model/manual/computed_column.md
new file mode 100644
index 0000000000..83b5296fa0
--- /dev/null
+++ b/website/docs/model/manual/computed_column.md
@@ -0,0 +1,187 @@
+---
+title: Computed Column
+language: en
+sidebar_label: Computed Column
+pagination_label: Computed Column
+toc_min_heading_level: 2
+toc_max_heading_level: 6
+pagination_prev: null
+pagination_next: null
+keywords:
+ - computed column
+draft: false
+last_update:
+ date: 09/13/2024
+---
+
+
+A **Computed Column** allows you to define actions such as data transformation and calculations directly within your models. This feature enhances data semantic abstraction tailored to various business scenarios. By shifting runtime calculations to offline index construction, Kylin effectively leverages its pre-calculation capabilities, resulting in significantly improved query performance. Additionally, computed columns support Hive UDFs, enabling the reuse of existing business logic and code.
+
+### <span id="concept">Basic Concepts</span>
+
+- **Naming Convention**: Computed column names may only include letters, digits, and underscores. Names cannot consist solely of digits or start with an underscore.
+
+- **Expression**: This refers to the logic used for calculations. The expression for a computed column can incorporate columns from both the fact table and a lookup table within the current model.
+
+- **Reuse and Reference**:
+ - Computed columns are tied to a specific model and fact table, meaning one expression cannot create multiple computed columns across different models with varying names.
+ - Strictly speaking, computed columns cannot be reused across models in the same project. To reuse a computed column in another model, you must define a new computed column with the same name and expression.
+ - Nested computed columns are supported, allowing you to utilize existing computed columns to create new ones.
+
+ - A single expression cannot generate multiple computed columns across different models that share the same fact table. For example, if you have two models, *model1* and *model2*, both using the fact table *LINEORDER*, and you wish to use the expression *LINEORDER.PRICE * LINEORDER.QUANTITY* to create computed columns in both models, the system will require you to use the same name, such as *T_PRICE*. Additionally, the name *T_PRICE* cannot be used for any other expressions within th [...]
+
+ :::tip Tips
+ - Computed columns can only be defined on the fact table. Once created, only the column expression can be modified.
+ - Avoid naming conflicts with other columns to prevent unexpected errors.
+ - Aggregate functions such as *SUM*, *MIN*, *MAX*, etc., are not permitted in computed column expressions.
+ - Creating a computed column with an expression that consists solely of constants is not recommended, such as *POWER(CAST(2 AS DOUBLE), 2)*.
+ - Avoid using keywords in computed column expressions. Refer to the [SQL Specification](../../query/specification/sql_spec.md) for a list of keywords.
+ - If an expression references identifiers that start with non-letter characters or include special characters, enclose the name in double quotes. For example: *"KYLIN_SALES"."100_PRICE" * "1_KYLIN_SALES"."ITEM_COUNT"*.
+ :::
+
+### <span id="create">Create</span>
+
+To help you master the creation of a computed column, we will demonstrate an example scenario below. Assume you have a fact table named *P_LINEORDER* with the following columns:
+
+- *LO_EXTENDEDPRICE*: Transaction price
+- *LO_QUANTITY*: Transaction quantity
+- *LO_ORDERDATE*: Transaction date
+
+We want to define two computed columns on this fact table: *T_PRICE_PER_ITEM* to calculate the total transaction price for each item, and *YEAR_OF_ORDER* to indicate the year of the order. The corresponding expressions are as follows:
+
+- *T_PRICE_PER_ITEM* ::= *P_LINEORDER.LO_EXTENDEDPRICE * P_LINEORDER.LO_QUANTITY*
+- *NEXT_ORDER_DAY* ::= *P_LINEORDER.LO_ORDERDATE + 1*
+
+**Step 1**: Click the button  marked in the picture. A window for **Computed Column** will pop up.
+
+
+
+**Step 2**: Click the button  in this window, and a dialog box titled **Add Computed Column** will appear. Please fill in the following information:
+
+- *Column name*: Defines the name of the computed column.
+- *Expression*: Specifies the calculation logic for the column.
+
+
+
+**Step 3**: Click the **Submit** button. The system will verify the legality of the name and expression of the computed column. If any issues arise, the system will provide feedback, allowing you to correct it and resubmit. Once the computed column is created, it will be visible in the fact table. For example, *T_PRICE_PER_ITEM* will appear in the fact table *P_LINEORDER*.
+
+
+
+**Step 4**: After creating the computed column, click the button  in the **Dimension** window to add a new dimension based on the computed column *NEXT_ORDER_DAY*, as shown below:
+
+
+
+Additionally, click the button  in the **Measure** window to add a new measure, *TOTAL_PRICE*, based on the computed column *T_PRICE_PER_ITEM*, as illustrated below:
+
+
+
+### <span id="edit">Modification</span>
+
+In certain situations, it may be necessary to change the expression to adapt to evolving business requirements. You can modify the expressions of computed columns directly by editing the model.
+
+
+
+:::note Warning
+However, there are a few limitations and considerations to keep in mind. Please review the following limitations carefully before using them:
+- Modifying the name of a computed column is not supported.
+- Modifying nested computed columns is not allowed. If a computed column has been utilized as a nested computed column, attempting to modify its expression will result in a failure, accompanied by the message: *model \[model_name\] nested computed column \[column_name\] still contains computed column \[column_name\]*.
+- Changes may require rebuilding **related indexes** under the model, prompting a message for user confirmation.
+- Changes may cause measures to become invalid, necessitating the deletion of related measures, aggregation groups, and layouts, which will also trigger a message for user confirmation.
+:::
+
+### <span id="index">Used in Index</span>
+
+Now that we have defined two computed columns in our model and created a new dimension and a new measure, we can leverage the pre-calculation capabilities of indexes. You can use computed columns in either an aggregate index or a table index. Let's walk through an example.
+
+
+
+After a successful submission, we have completed the basic usage of computed columns, including:
+
+- Creating a computed column.
+- Creating dimensions and measures based on computed columns.
+- Defining computed columns in the index.
+
+### <span id="query">Query</span>
+
+A computed column is logically appended to the table's column list after creation. You can query the computed column as if it were a normal column, provided it is pre-calculated in an index.
+
+To improve query performance with computed columns, ensure that you define them when creating an index.
+
+1. **Query Pushdown**
+
+ If a computed column is not used as a dimension or defined in indices, query performance will not be enhanced. However, if **QUERY PUSHDOWN** is enabled, users can still utilize this computed column. In this scenario, Kylin will analyze and translate the query into a SQL statement that the calculation engine can process.
+
+ For example, if a computed column named *T_PRICE_PER_ITEM* is defined with the expression *LO_EXTENDEDPRICE * LO_QUANTITY*, executing the following SQL:
+
+ ```sql
+ SELECT SUM(T_PRICE_PER_ITEM) FROM SSB.P_LINEORDER
+ ```
+
+ will be translated to:
+
+ ```sql
+ SELECT SUM(LO_EXTENDEDPRICE * LO_QUANTITY) FROM SSB.P_LINEORDER
+ ```
+
+ This query will then be pushed down to the calculation engine.
+
+ :::tip Tips
+ If you wish to query the computed column, you must supply the complete join relations defined in the model that contains the computed column in your SQL.
+ :::
+
+2. **Explicit Query**
+
+ If the name of a computed column appears as a field or a parameter of functions in a SQL statement, this is termed an **Explicit Query**. For example:
+
+ ```sql
+ SELECT SUM(T_PRICE_PER_ITEM) FROM SSB.P_LINEORDER
+ ```
+
+3. **Implicit Query:**
+
+ If the expression of a computed column appears as a field or a parameter of functions in a SQL statement, this is termed an **Implicit Query**. For example:
+
+ ```sql
+ SELECT SUM(LO_EXTENDEDPRICE * LO_QUANTITY) FROM SSB.P_LINEORDER
+ ```
+
+ In Kylin 4.x, the expression *LO_EXTENDEDPRICE * LO_QUANTITY* will be converted to *T_PRICE_PER_ITEM*. Consequently, the original query will be translated to:
+
+ ```sql
+ SELECT SUM(T_PRICE_PER_ITEM) FROM SSB.P_LINEORDER
+ ```
+
+ If the measure *SUM(T_PRICE_PER_ITEM)* has been pre-calculated in an Aggregate Index, query performance will be greatly enhanced.
+
+### <span id="nested">Nested Computed Columns</span>
+
+Computed columns can reference other computed columns as part of their expression, allowing for flexibility in data transformations. This means you can build complex calculations on top of existing ones. However, keep in mind the limitations regarding nested computed columns when modifying them, as detailed in the modification section.
+
+
+
+
+### <span id="faq">Frequently Asked Questions</span>
+
++ **QA: Can the computed columns be used in joins?**
+
+ Yes, computed columns can be utilized in joins, as long as they are defined in the model and pre-calculated in an index.
+
++ **QA: What happens if a computed column's expression references a non-existent column?**
+
+ The query will fail, and an error message will indicate that the column does not exist.
+
++ **QA: Are computed columns supported in all data types?**
+
+ Computed columns are limited to specific data types. Ensure compatibility when defining expressions.
+
++ **QA: How can I optimize the performance of queries using computed columns?**
+
+ To optimize performance, define computed columns on the model, use them in index and query them in your SQL to leverage pre-calculation benefits.
+
++ **QA: Can I use UDFs in computed column expressions?**
+
+ Yes, computed columns can utilize Hive UDFs, enabling the reuse of existing business logic.
+
+### <span id="summary">Summary</span>
+
+Computed columns in Kylin provide a powerful mechanism for enhancing data processing and query performance. By understanding their usage, properties, and potential limitations, you can leverage computed columns to optimize your analytics and data transformations effectively.
diff --git a/website/docs/model/manual/images/agg/add_agg_group.png b/website/docs/model/manual/images/agg/add_agg_group.png
new file mode 100644
index 0000000000..62b68f55f9
Binary files /dev/null and b/website/docs/model/manual/images/agg/add_agg_group.png differ
diff --git a/website/docs/model/manual/images/agg/advanced_settings.gif b/website/docs/model/manual/images/agg/advanced_settings.gif
new file mode 100644
index 0000000000..9194aad8a2
Binary files /dev/null and b/website/docs/model/manual/images/agg/advanced_settings.gif differ
diff --git a/website/docs/modeling/model_design/images/agg/agg_detail.png b/website/docs/model/manual/images/agg/agg_detail.png
similarity index 100%
rename from website/docs/modeling/model_design/images/agg/agg_detail.png
rename to website/docs/model/manual/images/agg/agg_detail.png
diff --git a/website/docs/model/manual/images/agg/agg_group_page.png b/website/docs/model/manual/images/agg/agg_group_page.png
new file mode 100644
index 0000000000..ec305bfdaa
Binary files /dev/null and b/website/docs/model/manual/images/agg/agg_group_page.png differ
diff --git a/website/docs/model/manual/images/agg/defined_agg_group.png b/website/docs/model/manual/images/agg/defined_agg_group.png
new file mode 100644
index 0000000000..31ed539c32
Binary files /dev/null and b/website/docs/model/manual/images/agg/defined_agg_group.png differ
diff --git a/website/docs/modeling/model_design/images/agg/AGG-4.png b/website/docs/model/manual/images/agg/examples/AGG-4.png
similarity index 100%
rename from website/docs/modeling/model_design/images/agg/AGG-4.png
rename to website/docs/model/manual/images/agg/examples/AGG-4.png
diff --git a/website/docs/modeling/model_design/images/agg/AGG-6.png b/website/docs/model/manual/images/agg/examples/AGG-6.png
similarity index 100%
rename from website/docs/modeling/model_design/images/agg/AGG-6.png
rename to website/docs/model/manual/images/agg/examples/AGG-6.png
diff --git a/website/docs/modeling/model_design/images/agg/Hierarchy-4.png b/website/docs/model/manual/images/agg/examples/Hierarchy-4.png
similarity index 100%
rename from website/docs/modeling/model_design/images/agg/Hierarchy-4.png
rename to website/docs/model/manual/images/agg/examples/Hierarchy-4.png
diff --git a/website/docs/modeling/model_design/images/agg/Mandatory-4.png b/website/docs/model/manual/images/agg/examples/Mandatory-4.png
similarity index 100%
rename from website/docs/modeling/model_design/images/agg/Mandatory-4.png
rename to website/docs/model/manual/images/agg/examples/Mandatory-4.png
diff --git a/website/docs/model/manual/images/agg/include_dimensions.png b/website/docs/model/manual/images/agg/include_dimensions.png
new file mode 100644
index 0000000000..a4a935b3e4
Binary files /dev/null and b/website/docs/model/manual/images/agg/include_dimensions.png differ
diff --git a/website/docs/model/manual/images/agg/include_measures.png b/website/docs/model/manual/images/agg/include_measures.png
new file mode 100644
index 0000000000..8ac3fecf2a
Binary files /dev/null and b/website/docs/model/manual/images/agg/include_measures.png differ
diff --git a/website/docs/model/manual/images/agg/index_view.png b/website/docs/model/manual/images/agg/index_view.png
new file mode 100644
index 0000000000..f6f02b0b0c
Binary files /dev/null and b/website/docs/model/manual/images/agg/index_view.png differ
diff --git a/website/docs/modeling/model_design/images/agg/AGG-2.png b/website/docs/model/manual/images/agg/principles/agg_group_p1.png
similarity index 100%
rename from website/docs/modeling/model_design/images/agg/AGG-2.png
rename to website/docs/model/manual/images/agg/principles/agg_group_p1.png
diff --git a/website/docs/modeling/model_design/images/agg/AGG-3.png b/website/docs/model/manual/images/agg/principles/agg_group_p2.png
similarity index 100%
rename from website/docs/modeling/model_design/images/agg/AGG-3.png
rename to website/docs/model/manual/images/agg/principles/agg_group_p2.png
diff --git a/website/docs/modeling/model_design/images/agg/Hierarchy-2.png b/website/docs/model/manual/images/agg/principles/hierarchy_p1.png
similarity index 100%
rename from website/docs/modeling/model_design/images/agg/Hierarchy-2.png
rename to website/docs/model/manual/images/agg/principles/hierarchy_p1.png
diff --git a/website/docs/modeling/model_design/images/agg/Hierarchy-3.png b/website/docs/model/manual/images/agg/principles/hierarchy_p2.png
similarity index 100%
rename from website/docs/modeling/model_design/images/agg/Hierarchy-3.png
rename to website/docs/model/manual/images/agg/principles/hierarchy_p2.png
diff --git a/website/docs/modeling/model_design/images/agg/AGG-5.png b/website/docs/model/manual/images/agg/principles/joint_p1.png
similarity index 100%
rename from website/docs/modeling/model_design/images/agg/AGG-5.png
rename to website/docs/model/manual/images/agg/principles/joint_p1.png
diff --git a/website/docs/modeling/model_design/images/agg/Mandatory-2.png b/website/docs/model/manual/images/agg/principles/mandatory_p1.png
similarity index 100%
rename from website/docs/modeling/model_design/images/agg/Mandatory-2.png
rename to website/docs/model/manual/images/agg/principles/mandatory_p1.png
diff --git a/website/docs/modeling/model_design/images/agg/Mandatory-3.png b/website/docs/model/manual/images/agg/principles/mandatory_p2.png
similarity index 100%
rename from website/docs/modeling/model_design/images/agg/Mandatory-3.png
rename to website/docs/model/manual/images/agg/principles/mandatory_p2.png
diff --git a/website/docs/modeling/model_design/images/mdc/index_mdc.png b/website/docs/model/manual/images/agg/principles/mdc_p1.png
similarity index 100%
rename from website/docs/modeling/model_design/images/mdc/index_mdc.png
rename to website/docs/model/manual/images/agg/principles/mdc_p1.png
diff --git a/website/docs/model/manual/images/agg/recognize.gif b/website/docs/model/manual/images/agg/recognize.gif
new file mode 100644
index 0000000000..a7a404ee0a
Binary files /dev/null and b/website/docs/model/manual/images/agg/recognize.gif differ
diff --git a/website/docs/model/manual/images/agg/to_agg_group_page.gif b/website/docs/model/manual/images/agg/to_agg_group_page.gif
new file mode 100644
index 0000000000..d12cb124f0
Binary files /dev/null and b/website/docs/model/manual/images/agg/to_agg_group_page.gif differ
diff --git a/website/docs/model/manual/images/computed_column/add_cc_icon.svg b/website/docs/model/manual/images/computed_column/add_cc_icon.svg
new file mode 100644
index 0000000000..d340715f55
--- /dev/null
+++ b/website/docs/model/manual/images/computed_column/add_cc_icon.svg
@@ -0,0 +1,4 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" viewBox="0 0 100 100">
+ <circle cx="50" cy="50" r="45" fill="gray" />
+ <text x="50%" y="50%" text-anchor="middle" dy=".35em" font-family="Arial" font-size="40" fill="white">CC</text>
+</svg>
diff --git a/website/docs/model/manual/images/computed_column/add_cc_page.png b/website/docs/model/manual/images/computed_column/add_cc_page.png
new file mode 100644
index 0000000000..4c8628422f
Binary files /dev/null and b/website/docs/model/manual/images/computed_column/add_cc_page.png differ
diff --git a/website/docs/model/manual/images/computed_column/add_dim_of_cc.png b/website/docs/model/manual/images/computed_column/add_dim_of_cc.png
new file mode 100644
index 0000000000..04d679f7e1
Binary files /dev/null and b/website/docs/model/manual/images/computed_column/add_dim_of_cc.png differ
diff --git a/website/docs/model/manual/images/computed_column/add_measure_of_cc.png b/website/docs/model/manual/images/computed_column/add_measure_of_cc.png
new file mode 100644
index 0000000000..996a94faae
Binary files /dev/null and b/website/docs/model/manual/images/computed_column/add_measure_of_cc.png differ
diff --git a/website/docs/model/manual/images/computed_column/cc_editing.gif b/website/docs/model/manual/images/computed_column/cc_editing.gif
new file mode 100644
index 0000000000..e19da525cb
Binary files /dev/null and b/website/docs/model/manual/images/computed_column/cc_editing.gif differ
diff --git a/website/docs/model/manual/images/computed_column/cc_for_index.gif b/website/docs/model/manual/images/computed_column/cc_for_index.gif
new file mode 100644
index 0000000000..a5c8dc4a50
Binary files /dev/null and b/website/docs/model/manual/images/computed_column/cc_for_index.gif differ
diff --git a/website/docs/model/manual/images/computed_column/cc_on_table.png b/website/docs/model/manual/images/computed_column/cc_on_table.png
new file mode 100644
index 0000000000..01b0a4938e
Binary files /dev/null and b/website/docs/model/manual/images/computed_column/cc_on_table.png differ
diff --git a/website/docs/model/manual/images/computed_column/nested_cc.png b/website/docs/model/manual/images/computed_column/nested_cc.png
new file mode 100644
index 0000000000..59ed91834f
Binary files /dev/null and b/website/docs/model/manual/images/computed_column/nested_cc.png differ
diff --git a/website/docs/model/manual/images/computed_column/where_is_cc.png b/website/docs/model/manual/images/computed_column/where_is_cc.png
new file mode 100644
index 0000000000..21b8f21c5f
Binary files /dev/null and b/website/docs/model/manual/images/computed_column/where_is_cc.png differ
diff --git a/website/docs/model/manual/images/modeling/add_dimension.gif b/website/docs/model/manual/images/modeling/add_dimension.gif
new file mode 100644
index 0000000000..241cd3d510
Binary files /dev/null and b/website/docs/model/manual/images/modeling/add_dimension.gif differ
diff --git a/website/docs/model/manual/images/modeling/add_join_relations.png b/website/docs/model/manual/images/modeling/add_join_relations.png
new file mode 100644
index 0000000000..35b8d2c5c7
Binary files /dev/null and b/website/docs/model/manual/images/modeling/add_join_relations.png differ
diff --git a/website/docs/model/manual/images/modeling/add_measure.gif b/website/docs/model/manual/images/modeling/add_measure.gif
new file mode 100644
index 0000000000..552fae5bc1
Binary files /dev/null and b/website/docs/model/manual/images/modeling/add_measure.gif differ
diff --git a/website/docs/modeling/images/add_model_set.png b/website/docs/model/manual/images/modeling/add_model_set.png
similarity index 100%
rename from website/docs/modeling/images/add_model_set.png
rename to website/docs/model/manual/images/modeling/add_model_set.png
diff --git a/website/docs/model/manual/images/modeling/added_tables.png b/website/docs/model/manual/images/modeling/added_tables.png
new file mode 100644
index 0000000000..ffd0dc87b3
Binary files /dev/null and b/website/docs/model/manual/images/modeling/added_tables.png differ
diff --git a/website/docs/model/manual/images/modeling/create_join_relations.gif b/website/docs/model/manual/images/modeling/create_join_relations.gif
new file mode 100644
index 0000000000..3678ef3b7c
Binary files /dev/null and b/website/docs/model/manual/images/modeling/create_join_relations.gif differ
diff --git a/website/docs/model/manual/images/modeling/data_loading_ways.png b/website/docs/model/manual/images/modeling/data_loading_ways.png
new file mode 100644
index 0000000000..620ea7d8b2
Binary files /dev/null and b/website/docs/model/manual/images/modeling/data_loading_ways.png differ
diff --git a/website/docs/model/manual/images/modeling/date_format.svg b/website/docs/model/manual/images/modeling/date_format.svg
new file mode 100644
index 0000000000..9d77303b17
--- /dev/null
+++ b/website/docs/model/manual/images/modeling/date_format.svg
@@ -0,0 +1,11 @@
+<svg class="icon" viewBox="0 0 1024 1024" version="1.1" xmlns="http://www.w3.org/2000/svg"
+ width="16" height="16">
+ <path d="M74.666667 405.333333a32 32 0 0 1 32-32h810.666666a32 32 0 0 1 32 32v298.666667a32 32 0 0 1-64 0v-266.666667H138.666667V874.666667c0 5.888 4.778667 10.666667 10.666666 10.666666H512a32 32 0 0 1 0 64H149.333333a74.666667 74.666667 0 0 1-74.666666-74.666666v-469.333334z"
+ fill="#363A3A"></path>
+ <path d="M149.333333 202.666667a10.666667 10.666667 0 0 0-10.666666 10.666666v160h746.666666V213.333333a10.666667 10.666667 0 0 0-10.666666-10.666666h-725.333334zM74.666667 213.333333c0-41.216 33.450667-74.666667 74.666666-74.666666h725.333334c41.216 0 74.666667 33.450667 74.666666 74.666666v192a32 32 0 0 1-32 32h-810.666666a32 32 0 0 1-32-32V213.333333z"
+ fill="#363A3A"></path>
+ <path d="M341.333333 74.666667a32 32 0 0 1 32 32v170.666666a32 32 0 0 1-64 0v-170.666666A32 32 0 0 1 341.333333 74.666667zM682.666667 74.666667a32 32 0 0 1 32 32v170.666666a32 32 0 0 1-64 0v-170.666666a32 32 0 0 1 32-32zM640 565.333333a117.333333 117.333333 0 1 0 0 234.666667 117.333333 117.333333 0 0 0 0-234.666667zM458.666667 682.666667a181.333333 181.333333 0 1 1 362.666666 0 181.333333 181.333333 0 0 1-362.666666 0z"
+ fill="#363A3A"></path>
+ <path d="M743.424 768.853333a32 32 0 0 1 45.056-4.096l128 106.666667a32 32 0 1 1-40.96 49.152l-128-106.666667a32 32 0 0 1-4.096-45.056z"
+ fill="#363A3A"></path>
+</svg>
diff --git a/website/docs/model/manual/images/modeling/finished_modeling.png b/website/docs/model/manual/images/modeling/finished_modeling.png
new file mode 100644
index 0000000000..d29df05761
Binary files /dev/null and b/website/docs/model/manual/images/modeling/finished_modeling.png differ
diff --git a/website/docs/model/manual/images/modeling/incremental_loading.png b/website/docs/model/manual/images/modeling/incremental_loading.png
new file mode 100644
index 0000000000..b788c02891
Binary files /dev/null and b/website/docs/model/manual/images/modeling/incremental_loading.png differ
diff --git a/website/docs/model/manual/images/modeling/joined_tables.png b/website/docs/model/manual/images/modeling/joined_tables.png
new file mode 100644
index 0000000000..39ccdc0f42
Binary files /dev/null and b/website/docs/model/manual/images/modeling/joined_tables.png differ
diff --git a/website/docs/model/manual/images/modeling/model_list.png b/website/docs/model/manual/images/modeling/model_list.png
new file mode 100644
index 0000000000..a118462854
Binary files /dev/null and b/website/docs/model/manual/images/modeling/model_list.png differ
diff --git a/website/docs/model/manual/images/modeling/modeling1.png b/website/docs/model/manual/images/modeling/modeling1.png
new file mode 100644
index 0000000000..b492d6302b
Binary files /dev/null and b/website/docs/model/manual/images/modeling/modeling1.png differ
diff --git a/website/docs/model/manual/images/modeling/redirect_to_add_index.png b/website/docs/model/manual/images/modeling/redirect_to_add_index.png
new file mode 100644
index 0000000000..9d51f39cc7
Binary files /dev/null and b/website/docs/model/manual/images/modeling/redirect_to_add_index.png differ
diff --git a/website/docs/model/manual/images/modeling/set_fact_table.png b/website/docs/model/manual/images/modeling/set_fact_table.png
new file mode 100644
index 0000000000..754d1759ea
Binary files /dev/null and b/website/docs/model/manual/images/modeling/set_fact_table.png differ
diff --git a/website/docs/model/manual/images/table_index/add_table_index.png b/website/docs/model/manual/images/table_index/add_table_index.png
new file mode 100644
index 0000000000..9b54e79616
Binary files /dev/null and b/website/docs/model/manual/images/table_index/add_table_index.png differ
diff --git a/website/docs/model/manual/images/table_index/index_overview.png b/website/docs/model/manual/images/table_index/index_overview.png
new file mode 100644
index 0000000000..5091aa7c6d
Binary files /dev/null and b/website/docs/model/manual/images/table_index/index_overview.png differ
diff --git a/website/docs/model/manual/images/table_index/table_index_list.png b/website/docs/model/manual/images/table_index/table_index_list.png
new file mode 100644
index 0000000000..5ed319433a
Binary files /dev/null and b/website/docs/model/manual/images/table_index/table_index_list.png differ
diff --git a/website/docs/model/manual/images/table_index/to_tableindex_page.gif b/website/docs/model/manual/images/table_index/to_tableindex_page.gif
new file mode 100644
index 0000000000..467b391582
Binary files /dev/null and b/website/docs/model/manual/images/table_index/to_tableindex_page.gif differ
diff --git a/website/docs/model/manual/modeling.md b/website/docs/model/manual/modeling.md
new file mode 100644
index 0000000000..6a1ab54f46
--- /dev/null
+++ b/website/docs/model/manual/modeling.md
@@ -0,0 +1,234 @@
+---
+title: Manual Modeling
+language: en
+sidebar_label: Manual Modeling
+pagination_label: Manual Modeling
+toc_min_heading_level: 2
+toc_max_heading_level: 6
+pagination_prev: null
+pagination_next: null
+keywords:
+ - manual modeling
+draft: false
+last_update:
+ date: 09/13/2024
+---
+
+This chapter explains how to design a manual model in Kylin. In this example, we use the [SSB dataset](../../quickstart/tutorial.md), which is based on real-world business applications, to analyze product and supplier information across various dimensions such as *year*, *city*, *supplier*, and *brand*.
+
+Before starting, ensure that you have created a project and loaded tables from the data source. If you haven't done so yet, please refer to the [Quickstart](../../quickstart/intro.md) chapter for detailed instructions.
+
+### Overview
+
+First, we will outline some key operations to ensure you have a clear understanding of the manual modeling process. These steps will help guide you through the creation and configuration of a model in Kylin.
+
+| **Main Step Content** | **Description** |
+|--------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| Define the fact table and dimension tables | Tables are the basis for data analysis. Kylin only supports one fact table, therefore kylin only support star or snowflake model. |
+| Create the join relations | Join relations is linked by the paired foreign keys and primary keys. Manual modeling only supports inner or left join. |
+| Add computed columns | Computed columns are expressions without aggregations. They can be utilized by dimensions and measures. More details please refer to [Computed Column](computed_column.md) |
+| Add the dimensions and measures | Dimensions and measures will be used for generating indexes. They constraint the scope of indexes. |
+| Specify the way of data loading | Choose increment data loading or full data loading. Increment loading needs setting date partition column. More details please refer to [Data Loading](../data_loading/intro.md) |
+| Save the model | Save the model and set the loading method. More details please refer to [Data Loading](../data_loading/intro.md) |
+
+
+### Create a Model
+
+Log in to Kylin with a role that has model management permissions. In the left navigation panel, click **Data Asset** > **Model**. You will then be directed to a page displaying the list of models, as shown below.
+
+
+
+Click the ** New** button to create a new model. In the pop-up dialog box, enter a name and description for the model, then click the **Create** button. You will be redirected to the model editing page where you can begin configuring your new model.
+
+
+
+:::tip Tips
+Model name can be any combination of numbers, letters, and underscores (_).
+:::
+
+
+### Add Tables
+
+On the model editing page, you can drag and drop tables onto the canvas. The system will automatically recommend the first table you drag as the fact table. You can either confirm this by selecting **Switch to Fact Table** or set the fact table later. In this example, the fact table is **P_LINEORDER**.
+
+
+
+
+:::tip Tips
+If there is no table in the **Data Source** on the model editing page, please [load data source](../../datasource/intro.md) first.
+:::
+
+You can create either a single-table model or a multi-table model. For a multi-table model, you'll need to add additional tables and define the join relationships between them. Besides the fact table, the other tables will serve as dimension tables, also known as lookup tables. Dimension tables typically store recurring attributes of the fact table, such as dates or geographic locations. Including dimension tables in the model helps reduce the size of the fact table and improves the effi [...]
+
+
+
+At this stage, the tables are ready to be joined. You can drag the foreign keys from the fact table to the corresponding primary keys in the dimension tables to establish the relationships. In a **star schema**, all foreign keys originate from the fact table, whereas in a **snowflake schema**, foreign keys can come from dimension tables as well. In this example, we are working with a star schema. The following GIF demonstrates the process in detail.
+
+
+
+In the **Add Join Relationship** dialog box, follow the steps below to configure the join relationships:
+
+- **Join Relationship for Tables**: This section contains three drop-down lists for specifying the primary table, foreign table, and join type. Kylin supports **LEFT** joins and **INNER** joins only.
+
+- **Table Relationship**: Defines the mapping between foreign keys and primary keys, which can be **One-to-One or Many-to-One**, **One-to-Many or Many-to-Many**.
+
+- **Precompute Join Relationship**: Indicates whether the join between the primary table and foreign table is precomputed. By default, the primary table is precomputed. For more information on this advanced feature, refer to [Runtime Join](../features/runtime_join.md).
+
+- **Join Relationship for Columns**: This section consists of three drop-down lists to define the primary column, foreign column, and comparison operator. You can specify multiple pairs of primary and foreign keys, which will be combined with an *AND* operator. The default comparison operator is **=**, indicating an equal join. For non-equal joins, refer to the advanced feature [SCD2](../features/scd2.md) for further details.
+
+
+
+
+
+:::tip Tips
+Join relations should meet the following requirements:
+- Do not define more than one join relation for the same column; two tables could only be joined by the same condition for one time
+- Join relations for columns should include at least one equal-join condition (=)
+- Join relations ≥ and < must be used in pairs, and the column in between must be the same. Example: B ≥ A, C < A
+:::
+
+After the tedious process, you can get a model with four join relations as shown below.
+
+
+
+This model can also be created directly using the following SQL statement. While this approach is straightforward, it leverages a new feature. For more information, please refer to [Modeling by SQL](../rec/sql_modeling.md).
+
+```sql
+SELECT
+ 1
+FROM SSB.P_LINEORDER
+ LEFT JOIN SSB.DATES ON P_LINEORDER.LO_ORDERDATE = DATES.D_DATEKEY
+ LEFT JOIN SSB.CUSTOMER ON P_LINEORDER.LO_CUSTKEY = CUSTOMER.C_CUSTKEY
+ LEFT JOIN SSB.SUPPLIER ON P_LINEORDER.LO_SUPPKEY = SUPPLIER.S_SUPPKEY
+ LEFT JOIN SSB.PART ON P_LINEORDER.LO_PARTKEY = PART.P_PARTKEY
+```
+
+### Create Computed Columns
+
+This section of content is optional and may seem a bit complex, so we will cover it in the chapter of [Computed Column](computed_column.md).
+
+
+### Add or Delete Dimensions
+On the model editing page, there are two ways to add dimensions:
+1. Drag the desired column from the joined tables to the **Dimension** card.
+2. Click the  button in the **Dimension** card. When you hover over the  button, a tooltip labeled **Batch Add** appears, indicating that you can add multiple dimensions simultaneously.
+
+The following GIF illustrates these ways in detail.
+
+
+
+By default, the dimension name matches the column name. However, if duplicate columns share the same name, the system will automatically modify the dimension name by appending the table alias, separated by an underscore (*_*). Additionally, you can manually specify the dimension name, which can include any combination of letters, numbers, spaces, and special characters such as `_ -()%?`.
+
+In our example, we add the columns *LO_CUSTKEY* and *LO_SUPPKEY* from the table *P_LINEORDER* as dimensions.
+
+Deleting dimensions is similar to adding them. However, if a dimension is being used by any indexes, you must delete the associated indexes first. Otherwise, the deletion operation will fail.
+
+#### Add or Delete Measures
+
+On the model editing page, there are three ways to add measures:
+1. Drag the desired column from the joined tables to the **Measure** card.
+2. Click the left  button in the **Measure** card. When you hover over the  button, a tooltip labeled **Add** appears, indicating that you can add one measure. Click the  button, it will pop up the **Add Measure** dialog box to guide you what to do next.
+3. Click the middle  button in the **Measure** card. When you hover over the  button, a tooltip labeled **Batch Add** appears, indicating that you can add multiple measures. Click it, it will pop up the **Quick Add Measure** dialog box to let you define measures more easily.
+
+The following GIF illustrates these ways in detail.
+
+
+
+
+In the **Add Measure** dialog box, you can adjust the following properties if needed:
+
+- **Name**: By default, it is the column name. It can be any combination of letters, numbers, spaces, and special characters such as `_ -()%?`.
+- **Function**: Defaults to *SUM (column)*. Kylin supports a variety of built-in aggregation functions, including Count Distinct and TopN. If you want to know advanced measures like *TopN*, refer to [Advanced measures](../features/measures/intro.md).
+- **Column**: Specifies the column of measure. If using the first way, this is set automatically. If you're using the second way, you'll need to select the column that serves as the parameter for the measure.
+- **Comment** (Optional): Adding a comment to explain the measure’s purpose can be helpful for who cares about the meaning.
+
+In the **Quick Add Measure** dialog box, you can create measures using *SUM*, *COUNT*, *MIN*, *MAX*, and *COUNT DISTINCT* functions. When using this option, the measure name is generated automatically, and you cannot modify the name or add a comment.
+
+In our example, we added three measures to the model:
+- *SUM(LO_QUANTITY)* with the name **LO_QUANTITY**.
+- *SUM(LO_EXTENDEDPRICE)* with the name **mea1**.
+- *MAX(LO_ORDERDATE)* with the name **LO_ORDERDATE_MAX**.
+
+:::tip Warning
+Ensure that the measure name is different from the column name, as this may cause issues when integrating with BI tools.
+:::
+
+Deleting measures is similar to adding them. However, if a measure is being used by any indexes, you must delete the associated indexes first. Otherwise, the deletion operation will fail.
+
+### Save Model
+
+When saving the model, there are several steps to be completed: setting the data loading method, configuring the data filter condition, and adding base indexes.
+
+#### Set the Data Loading Way
+
+After adding dimensions and measures, you need to set the data loading method by clicking the **Save** button in the upper right corner of the model editing page. When you click this button, the **Save Model** dialog box will appear. In this dialog box, you'll be prompted to choose between **Incremental Load Recommended** and **Full Load** options for data loading.
+
+
+
+- **Full Load**: This option loads and pre-computes all data from the source table based on the different combinations of dimensions and measures.
+
+- **Incremental Load**: This option loads and pre-computes only the data within a specified time range, according to the combinations of dimensions and measures. If this option is selected, you will need to specify the following parameters:
+ - **Partition Table**: The fact table (default and cannot be changed).
+ - **Time Partition Column**: Select a column of the time type from the partition table.
+ - **Time Format**: Choose the appropriate time format, or click the  icon, and Kylin will automatically detect and fill in the time format.
+
+
+
+
+#### Data Filter Condition
+
+As shown in the picture above, there is an **Advanced Setting** card. By expanding this card, you can set the **data filter condition**, which is useful for filtering out null values or data that meets specific criteria. You can use the *AND* and *OR* operators to combine multiple filters. For example, you might use a condition like *BUYER_ID \<> 0001 AND COUNT_ITEM > 1000 OR TOTAL_PRICE = 1000*.
+
+
+#### Base Indexes
+
+Base indexes are beneficial for business scenarios where users are aware of the dimensions and measures but may not know how to organize them into indexes by aggregate groups. Adding base indexes to the model is optional and is disabled by default. You can enable this feature by checking the boxes next to **Base Aggregate Index** and **Base Table Index**.
+
+- **Base Aggregate Index**: This index includes all model dimensions and measures.
+- **Base Table Index**: This index includes all columns associated with the model dimensions and measures.
+
+
+### Jump to Model List
+
+After clicking the **Save** button, a dialog box will appear, allowing you to add indexes.
+
+
+
+You can select the **Not Now** button to be redirected to the **Model List** page, as shown below.
+
+
+
+### Summary
+
+We have now completed the model design. The next step is to add [aggregate indexes](aggregation_group.md) and [table indexes](table_index.md), and then proceed to [build them](../data_loading/intro.md). After completing these steps, you can query the **Insight** page, where the SQL generated by the model will be displayed on the results page.
+
+
+### FAQ
+
+- Question: Why did I get an error when saving the time partition column settings?
+
+ Answer: This error occurs when the time format of the time partition column does not match the target format. Kylin supports the following time formats: *yyyyMMdd*, *yyyy-MM-dd*, *yyyy/MM/dd*, *yyyy-MM-dd HH:mm:ss*, *yyyy-MM-dd HH:mm:ss.SSS*, *yyyy-MM*, *yyyyMM* and *yyyy-MM-dd'T'HH:mm:ss.SSS'Z'*.
+
+ Kylin also supports customized time formats if the following conditions are met:
+
+ - Any combination of yyyy, MM, dd, HH, mm, ss, and SSS is used with the elements in ascending order.
+ - Hyphens (-), forward slashes (/), columns (:), or spaces are used as separators.
+ - Unformatted letters are enclosed in single quotation marks ('). For example, 'T' is recognized as T.
+
+ :::note
+ When the time format is customized as *yyyyMMddHHmmss*, the corresponding column in the Hive table should be strings, or Kylin may fail to recognize column data.
+ :::
+
+- Question: I've modified several tables from the same model. Why did I get an error when reloading these tables?
+
+ Answer: Since Kylin currently only supports loading a single table at a time, please edit and reload tables one by one, rather than reload several modified tables at a time.
+
+- Question: Why did I get an error when probing the time partition format?
+
+ Answer: Since Kylin currently supports probing the time format of the partition column only for tables, not for views in Hive, probing views may lead to Out of Memory (OOM) error.
+
+- Question: What are the rules for the model to go online?
+
+ **Answer**: The model will automatically switch to online status upon the completion of the building job. However, customers may want to continue building historical data for a period while creating a new report or during testing, and they may prefer that the model does not serve any queries until all data is built. To accommodate this need, we provide a model-level configuration that allows users to control the model status manually. By setting `kylin.model.offline=true` (the default [...]
+
+ 
diff --git a/website/docs/model/manual/table_index.md b/website/docs/model/manual/table_index.md
new file mode 100644
index 0000000000..eb59533d8f
--- /dev/null
+++ b/website/docs/model/manual/table_index.md
@@ -0,0 +1,59 @@
+---
+title: Table Index
+language: en
+sidebar_label: Table Index
+pagination_label: Table Index
+toc_min_heading_level: 2
+toc_max_heading_level: 6
+pagination_prev: null
+pagination_next: null
+keywords:
+ - table index
+draft: false
+last_update:
+ date: 09/14/2024
+---
+
+### Overview
+
+In the previous chapter, we discussed how to create an Aggregate Group. Now, we'll focus on its counterpart, the Table Index. A Table Index looks like a table in a data source, but unlike standard tables, it is derived from a flat-table resulting from a join operation. The columns used in a Table Index are from dimensions and measures already defined in the model.
+
+- **Advantages**: Eliminates the overhead of joining tables at query time, resulting in faster query performance.
+
+- **Disadvantages**: When the join relationship is one-to-many, the amount of data can increase significantly. To solve this issue, please refer to the [runtime join](../features/runtime_join.md) chapter for more details.
+
+Now, let's introduce how to create a Table Index. The following GIF demonstrates how to navigate to the **Table Index** page.
+
+
+
+### Create Table Index
+
+Click the  button on the Table Index page to open a new window titled **Add Table Index**. This window allows you to create a table index by selecting the columns to include or searching the columns quickly. The available name of columns, along with their types, are displayed on this page.
+
+
+
+After selecting the columns, you can set the shardBy columns and adjust the column order. Choosing an appropriate ShardBy column can distribute raw data into multiple shards, which helps increase concurrency and improve query performance. It's recommended to select columns with relatively large cardinality as the shardBy column to avoid uneven data distribution. Similar to aggregate groups, only one shardBy column will be utilized during queries, and this is optional for the table index too.
+
+You should also arrange the columns in the order of their frequency as filter conditions, as this arrangement can significantly impact query performance.
+
+Currently, you can only add one table index at a time. If you wish to add another, click the  button on the Table Index page and repeat the steps above. Please note that having more table indexes will increase job building and storage costs.
+
+### Table Index View
+
+After creating the table index, you can return to the **Table Index** page to view the indexes you just added. In this example, we successfully added two as follows.
+
++ \[*LO_ORDERDATE*, *LO_SUPPKEY*, *LO_QUANTITY*, *LO_EXTENDEDPRICE*\], *LO_ORDERDATE* as shardBy column
++ \[*D_DATE*, *LO_ORDERDATE*, *LO_CUSTKEY*, *D_MONTH*, *D_YEAR*, *LO_EXTENDEDPRICE*\]
+
+
+
+
+### Index Overview
+
+If you want to know the indexes generated by the aggregate group, you should navigate to the **Index Overview** page. This page gives you all the information of index, we will introduce them in the chapter of [model info](http://localhost:3000/latest/docs/model/manage/model_info).
+
+
+
+### Summary
+
+In this chapter, we covered how to create table index. However, designing effective indexes seems quite complex, especially for beginners. If you're looking for a simpler approach, please refer to the [recommendation](../rec/intro.md) for more details.
diff --git a/website/docs/modeling/auto_modeling/basic_concept_actions.md b/website/docs/model/rec/basic_concept_actions.md
similarity index 97%
rename from website/docs/modeling/auto_modeling/basic_concept_actions.md
rename to website/docs/model/rec/basic_concept_actions.md
index 60b39d0f85..1798371687 100644
--- a/website/docs/modeling/auto_modeling/basic_concept_actions.md
+++ b/website/docs/model/rec/basic_concept_actions.md
@@ -5,7 +5,7 @@ sidebar_label: Intelligent Recommendation Principles
pagination_label: Intelligent Recommendation Principles
toc_min_heading_level: 2
toc_max_heading_level: 6
-pagination_prev: modeling/auto_modeling/intro
+pagination_prev: null
pagination_next: null
keywords:
- recommendation
@@ -22,7 +22,7 @@ Kylin automatically parses your SQL statements for dimensions, measures, and oth
You can enter SQL statements in either of the following ways:
-- [Web UI](data_modeling_by_SQL.md): Upload the SQL statements as a TXT or SQL file.
+- [Web UI](images/sql_modeling): Upload the SQL statements as a TXT or SQL file.
- [API](../../restapi/model_api/intro.md#sql): Call an API to upload the SQL statements as parameter values.
diff --git a/website/docs/modeling/auto_modeling/images/flexible_index.png b/website/docs/model/rec/images/flexible_index.png
similarity index 100%
rename from website/docs/modeling/auto_modeling/images/flexible_index.png
rename to website/docs/model/rec/images/flexible_index.png
diff --git a/website/docs/modeling/auto_modeling/images/low_usage.png b/website/docs/model/rec/images/low_usage.png
similarity index 100%
rename from website/docs/modeling/auto_modeling/images/low_usage.png
rename to website/docs/model/rec/images/low_usage.png
diff --git a/website/docs/model/rec/images/qh_recommend/adjust_rec_conf.png b/website/docs/model/rec/images/qh_recommend/adjust_rec_conf.png
new file mode 100644
index 0000000000..6f4522c7ec
Binary files /dev/null and b/website/docs/model/rec/images/qh_recommend/adjust_rec_conf.png differ
diff --git a/website/docs/model/rec/images/qh_recommend/rec_rules.png b/website/docs/model/rec/images/qh_recommend/rec_rules.png
new file mode 100644
index 0000000000..e92abd8095
Binary files /dev/null and b/website/docs/model/rec/images/qh_recommend/rec_rules.png differ
diff --git a/website/docs/model/rec/images/qh_recommend/recommendations.png b/website/docs/model/rec/images/qh_recommend/recommendations.png
new file mode 100644
index 0000000000..16a68f2919
Binary files /dev/null and b/website/docs/model/rec/images/qh_recommend/recommendations.png differ
diff --git a/website/docs/modeling/auto_modeling/images/recommendation_detail_en.png b/website/docs/model/rec/images/recommendation_detail_en.png
similarity index 100%
rename from website/docs/modeling/auto_modeling/images/recommendation_detail_en.png
rename to website/docs/model/rec/images/recommendation_detail_en.png
diff --git a/website/docs/modeling/auto_modeling/images/recommendation_preferences_en.png b/website/docs/model/rec/images/recommendation_preferences_en.png
similarity index 100%
rename from website/docs/modeling/auto_modeling/images/recommendation_preferences_en.png
rename to website/docs/model/rec/images/recommendation_preferences_en.png
diff --git a/website/docs/modeling/auto_modeling/images/recommendations.en.png b/website/docs/model/rec/images/recommendations.en.png
similarity index 100%
rename from website/docs/modeling/auto_modeling/images/recommendations.en.png
rename to website/docs/model/rec/images/recommendations.en.png
diff --git a/website/docs/model/rec/images/sql_modeling/convert_success.png b/website/docs/model/rec/images/sql_modeling/convert_success.png
new file mode 100644
index 0000000000..be7addc283
Binary files /dev/null and b/website/docs/model/rec/images/sql_modeling/convert_success.png differ
diff --git a/website/docs/model/rec/images/sql_modeling/convert_to_index.png b/website/docs/model/rec/images/sql_modeling/convert_to_index.png
new file mode 100644
index 0000000000..3a9f9ea577
Binary files /dev/null and b/website/docs/model/rec/images/sql_modeling/convert_to_index.png differ
diff --git a/website/docs/model/rec/images/sql_modeling/create_new_model.png b/website/docs/model/rec/images/sql_modeling/create_new_model.png
new file mode 100644
index 0000000000..b6305c988e
Binary files /dev/null and b/website/docs/model/rec/images/sql_modeling/create_new_model.png differ
diff --git a/website/docs/model/rec/images/sql_modeling/create_or_convert.png b/website/docs/model/rec/images/sql_modeling/create_or_convert.png
new file mode 100644
index 0000000000..563d5ad542
Binary files /dev/null and b/website/docs/model/rec/images/sql_modeling/create_or_convert.png differ
diff --git a/website/docs/model/rec/images/sql_modeling/from_sql.png b/website/docs/model/rec/images/sql_modeling/from_sql.png
new file mode 100644
index 0000000000..16cb9f40b3
Binary files /dev/null and b/website/docs/model/rec/images/sql_modeling/from_sql.png differ
diff --git a/website/docs/model/rec/images/sql_modeling/imported_sql.png b/website/docs/model/rec/images/sql_modeling/imported_sql.png
new file mode 100644
index 0000000000..5a44a5e8d0
Binary files /dev/null and b/website/docs/model/rec/images/sql_modeling/imported_sql.png differ
diff --git a/website/docs/model/rec/images/sql_modeling/left_join_model.png b/website/docs/model/rec/images/sql_modeling/left_join_model.png
new file mode 100644
index 0000000000..99e9caa0ca
Binary files /dev/null and b/website/docs/model/rec/images/sql_modeling/left_join_model.png differ
diff --git a/website/docs/model/rec/images/sql_modeling/model_list.png b/website/docs/model/rec/images/sql_modeling/model_list.png
new file mode 100644
index 0000000000..b4a8eba66d
Binary files /dev/null and b/website/docs/model/rec/images/sql_modeling/model_list.png differ
diff --git a/website/docs/model/rec/images/sql_modeling/preview_models.png b/website/docs/model/rec/images/sql_modeling/preview_models.png
new file mode 100644
index 0000000000..5bee1a8c0d
Binary files /dev/null and b/website/docs/model/rec/images/sql_modeling/preview_models.png differ
diff --git a/website/docs/model/rec/images/sql_modeling/show_accepted_index.png b/website/docs/model/rec/images/sql_modeling/show_accepted_index.png
new file mode 100644
index 0000000000..473cd5732d
Binary files /dev/null and b/website/docs/model/rec/images/sql_modeling/show_accepted_index.png differ
diff --git a/website/docs/model/rec/images/sql_modeling/upload_sql.png b/website/docs/model/rec/images/sql_modeling/upload_sql.png
new file mode 100644
index 0000000000..2eaa936709
Binary files /dev/null and b/website/docs/model/rec/images/sql_modeling/upload_sql.png differ
diff --git a/website/docs/model/rec/images/turn_on_recommendation.gif b/website/docs/model/rec/images/turn_on_recommendation.gif
new file mode 100644
index 0000000000..9101725b82
Binary files /dev/null and b/website/docs/model/rec/images/turn_on_recommendation.gif differ
diff --git a/website/docs/modeling/auto_modeling/optimize_index/index_optimization.md b/website/docs/model/rec/index_optimization.md
similarity index 99%
rename from website/docs/modeling/auto_modeling/optimize_index/index_optimization.md
rename to website/docs/model/rec/index_optimization.md
index 124f79d38b..96304facc4 100644
--- a/website/docs/modeling/auto_modeling/optimize_index/index_optimization.md
+++ b/website/docs/model/rec/index_optimization.md
@@ -11,7 +11,7 @@ keywords:
- index optimization
draft: false
last_update:
- date: 09/02/2024
+ date: 09/14/2024
---
Kylin recommendation engine includes index optimization as one of its core capabilities. Besides recommending new high value indexes to reduce query response times, Kylin also recommends deleting low value indexes discovered following index optimization strategies to help reduce index building pressure and storage costs. This article introduces index optimization strategies.
@@ -52,7 +52,7 @@ The Kylin recommendation engine automatically analyzes how often an index is hit
To adjust the low-frequency strategy, log in to Kylin, click the project **Setting **page, and modify the strategy in the **Index Optimization** section.
-
+
### **Apply index optimization strategies**
diff --git a/website/docs/model/rec/intro.md b/website/docs/model/rec/intro.md
new file mode 100644
index 0000000000..1cf5ceac2c
--- /dev/null
+++ b/website/docs/model/rec/intro.md
@@ -0,0 +1,32 @@
+---
+title: Recommendation
+language: en
+sidebar_label: Recommendation
+pagination_label: Recommendation
+toc_min_heading_level: 2
+toc_max_heading_level: 6
+pagination_prev: null
+pagination_next: null
+keywords:
+ - recommendation
+draft: false
+last_update:
+ date: 09/02/2024
+---
+
+Following the manual modeling section, it’s evident that the work involved can be tedious and labor-intensive, relying heavily on the expertise of data modelers. In this chapter, we will introduce an easier way to accomplish this work in Kylin 5.0 through a feature called **Recommendation**. Note that this type of recommendation differs from the its counterpart in machine learning, but it significantly improves your efficiency. We will cover the following topics:
+
+- **[Imported SQL Modeling](sql_modeling.md)**: The Kylin recommendation engine can recognize input SQL statements, extract dimensions, measures, and other critical information, and automatically translate this data into models and indexes. This means you do not need in-depth knowledge to create models that meet your needs, allowing you to focus more on your business.
+
+- **[Query History Recommendation](optimize_by_qh.md)**: The Kylin recommendation engine periodically analyzes query history records and transforms them into recommendations. You just need to review the proposed recommendations and make any necessary adjustments to the configuration to achieve your desired outcomes. Once you approve them, query performance can be significantly accelerated.
+
+- **[Index Optimization](index_optimization.md)**: The Kylin recommendation engine analyzes the relationships among existing indexes to identify those that are useless or ineffective. It then generates recommendations for deletion, helping you save time on building and managing indexes as well as conserving storage space.
+
+Before the journey, you should turn on the recommendation function in the project setting page.
+
+
+
+Now, let’s embark on this journey.
+
+
+
diff --git a/website/docs/model/rec/optimize_by_qh.md b/website/docs/model/rec/optimize_by_qh.md
new file mode 100644
index 0000000000..cfad9d86a7
--- /dev/null
+++ b/website/docs/model/rec/optimize_by_qh.md
@@ -0,0 +1,136 @@
+---
+title: Query History Recommendation
+language: en
+sidebar_label: Query History Recommendation
+pagination_label: Query History Recommendation
+toc_min_heading_level: 2
+toc_max_heading_level: 6
+pagination_prev: null
+pagination_next: null
+keywords:
+ - optimization
+draft: false
+last_update:
+ date: 09/18/2024
+---
+
+In the previous chapter, we introduced [imported SQL modeling](sql_modeling.md). Here, we will explore another feature—*
+*optimizing indexes using query history** in the Kylin Engine. This capability allows the system to recognize and
+propose the most valuable indexes based on past queries, helping to accelerate query performance. It simplifies the
+process of selecting indexes.
+
+This chapter will cover the following:
+
+1. **Settings to Enable Query History Optimization**: How to configure the system to recognize valuable indexes from
+ query history.
+2. **Generating Recommendations**: The process for generating index recommendations based on historical query data.
+3. **Approving Recommendations**: How to review and approve these recommendations to create the appropriate indexes for
+ optimized performance.
+
+### Peferences
+
+After enable the recommendation engine, the **Recommendation Preferences** panel will appear, the page as follows. You
+can set user rules, duration rules, hit rules, and limit of Recommendations for Adding Index.
+
++ **User Rules:** Turn on the toggle switch if you only want to generate recommendations for selected users and/or user
+ groups. Then you can select users and/or user groups in the following field(s).
+
++ **Duration Rule:** Generate recommendations for queries within certain duration. Then you can customize the query
+ duration. Default: 5 to 3600 second(s).
+
+
+- **Hit Rules**: Set the parameters below to filter out indexes that are frequently hit by queries, that is, the indexes
+ of high value.
+ For example, if the **Time Window** is set to 2 and **Number of Hits** is set to 30, the system will only prompt
+ recommendations with at least 30 query hits in the past 2 days.
+ - **Time Window**: Set the number of days to be calculated for the query hits, starting from today, default: 2 days.
+ - **Number of Hits**: Set the number of query hits (default: 30 hits). This hit count can later be modified based on
+ the recommendations.
+
+- **Limit of Recommendations for Adding Index**: Set how many recommendations will be prompted and the push frequency.
+ This parameter is only valid for recommendations based on query history.
+ - **Number of recommendations**: Set the number of recommendations that will be pushed. Recommendations will be
+ ranked based on the query hit rates (from highest to lowest); the default value is 20, indicating only the top 20
+ recommendations will be prompted.
+ - **Recommendation Frequency**: Set the frequency (in days) to push the recommendations, default: 2 days.
+
+
+
+### Generate
+
+The generation of index suggestions is handled as a background task. This task runs at regular intervals, generating
+recommendations and storing them in the recommendation table of the database. At this stage, all suggestions remain
+invisible. When multiple query histories result in the same recommendation, the system accumulates the recommendation's
+statistical frequency on a daily basis.
+
+To configure the task's running frequency, you can refer to the following configuration items. These settings allow you
+to control how often the system scans query history and updates recommendations based on usage patterns.
+
+```shell
+# This parameter is used for controlling the maximum queries
+# processed by each scheduled task.
+# Since the query history table may contain too many records,
+# each task will process at most 1000000 queries by default.
+# The engine does not handle all query history items in a single
+# batch due to memory constraints.
+# Instead, it processes them in smaller batches. By default,
+# each batch handles 1,000 queries.
+# The task interval is 60 minutes by default.
+
+kylin.favorite.query-history-accelerate-max-size=1000000
+kylin.favorite.query-history-accelerate-batch-size=1000
+kylin.favorite.query-history-accelerate-interval=60m
+```
+
+In addition, there is a scheduled task in the system that calculates how often a recommended index is "hit" during a
+specific **Time Window**. When the **Number of Hits** exceeds the configured threshold, the system exposes the
+recommendation to the user, who can then choose whether to accept it. The frequency at which the system generates these
+suggestions is controlled by the **Recommendation Frequency** configuration.
+
+:::tip Tips
+Ensure that you have sufficient memory available before modifying the configurations mentioned above.
+Complex queries typically demand more memory, so if needed, you can adjust the memory limit by modifying
+the `conf/setenv.sh` file.
+
+- export JAVA_VM_TOOL_XMS=1g
+- export JAVA_VM_TOOL_XMX=8g
+ :::
+
+To quickly verify the ability of the system to generate index recommendations, follow these steps:
+
+1. **Execute the Query**: Go to the **Insight** page and run many query statements.
+
+2. **Adjust Configuration Parameters**: Modify the recommended configuration settings to match the values shown in the
+ figure.
+
+
+
+
+By running the query and adjusting the parameters, the executing the following API request to trigger the generation of recommendations. After executing the request successfully, you should be able to see recommendations generated based on query history.
+```shell
+curl -X PUT 'http://localhost:7070/kylin/api/recommendations/acceleration?project=learn_kylin' \
+ -H 'Accept-Encoding: gzip, deflate, br' \
+ -H 'Accept: application/vnd.apache.kylin-v4+json' \
+ -H 'Content-Type: application/json;charset=UTF-8' \
+ -H 'Accept-Language: en'
+```
+
+
+
+### Approve
+
+On the **Recommendation** page, you have the option to review and manage the suggested indexes. You can take the following actions:
+
+- **Accept**: If you choose to accept the recommendation, it will be converted into an index. The index will be applied to the model to improve query performance.
+
+- **Delete**: If you choose to delete the recommendation, it will not be physically removed from the system but will be marked as deleted (logical deletion) and remain in the recommendation table for record-keeping purposes.
+
+
+
+### **FAQ**
+
+- Question: At which level will the **Recommendations Preferences** be effective?
+
+ Answer: At the project level. To exclude certain tables at the model level, that is, this table will not be calculated
+ during the pre-computation and indexing, please deselect the **Precompute Join Relationships** checkbox in the **Add
+ Join Relationship** dialog box.
diff --git a/website/docs/modeling/auto_modeling/optimize_index/intro.md b/website/docs/model/rec/optimize_index/intro.md
similarity index 100%
rename from website/docs/modeling/auto_modeling/optimize_index/intro.md
rename to website/docs/model/rec/optimize_index/intro.md
diff --git a/website/docs/modeling/auto_modeling/optimize_index/review_index.md b/website/docs/model/rec/optimize_index/review_index.md
similarity index 76%
rename from website/docs/modeling/auto_modeling/optimize_index/review_index.md
rename to website/docs/model/rec/optimize_index/review_index.md
index 2476bff0d8..d8fb50ad96 100644
--- a/website/docs/modeling/auto_modeling/optimize_index/review_index.md
+++ b/website/docs/model/rec/optimize_index/review_index.md
@@ -27,7 +27,7 @@ In traditional model development mode, data modelers may need to frequently upda
2. In the left navigation panel, click **Data Assets** > **Model**.
-3. Check if there is any recommendation for the model. Click  to view details about the recommendation.
+3. Check if there is any recommendation for the model. Click to view details about the recommendation.
@@ -35,21 +35,20 @@ In traditional model development mode, data modelers may need to frequently upda
- In this example, Kylin recommends adding an aggregate index. Click in the **Content** column to check the recommended dimensions and measures. If the recommendation type is **Delete**, you can check the reason in the **Note** column to evaluate whether to approve this recommendation.
+ In this example, Kylin recommends adding an aggregate index. Click in the **Content** column to check the recommended dimensions and measures. If the recommendation type is **Delete**, you can check the reason in the **Note** column to evaluate whether to approve this recommendation.
-5. Select the check box before the recommendation and click **Approve**.
+5. Select the checkbox before the recommendation and click **Approve**.
6. After adding the new index, you still need to build it so Kylin can help to accelerate queries with beforehand precomputations based on the index.
1. Click the **Index Overview** tab.
- 2. Find the index just created. Click  in the **Actions** column.
+ 2. Find the index just created. Click  in the **Actions** column.
- 
You can also use the search filter to quickly locate the indexes in **NO BUILD** status.
-
- 3. In the pop-up dialog box, select the check box before time range, and click **Build Index**.
+
+ 3. In the pop-up dialog box, select the checkbox before time range, and click **Build Index**.
To check the progress of the index building task, click **Monitor** > **Batch Job** in the left navigation panel.
@@ -61,4 +60,4 @@ In traditional model development mode, data modelers may need to frequently upda
- Question: What index optimization strategies are supported?
- Answer: Kylin supports the low-frequency, inclusion, and similarity strategies to help you reduce build pressure and storage cost. For details, see [Index Optimization Strategies](index_optimization.md).
\ No newline at end of file
+ Answer: Kylin supports the low-frequency, inclusion, and similarity strategies to help you reduce build pressure and storage cost. For details, see [Index Optimization Strategies](index_optimization.md).
diff --git a/website/docs/model/rec/rule_setting.md b/website/docs/model/rec/rule_setting.md
new file mode 100644
index 0000000000..9be5658c7f
--- /dev/null
+++ b/website/docs/model/rec/rule_setting.md
@@ -0,0 +1,74 @@
+---
+title: Set Recommendation Rules
+language: en
+sidebar_label: Set Recommendation Rules
+pagination_label: Set Recommendation Rules
+toc_min_heading_level: 2
+toc_max_heading_level: 6
+pagination_prev: null
+pagination_next: null
+keywords:
+ - recommendation
+ - rule
+draft: false
+last_update:
+ date: 09/02/2024
+---
+
+Kylin optimizes query performance with index or model recommendations based on query history, data characteristics, SQL modeling, etc. The [Recommendation](intro.md) feature can simplify model building and accelerate queries. This article covers how to configure the setting for the recommendations.
+
+### **Prerequisites**
+
+Please make sure that the **Recommendation** feature is turned on.
+
+:::info
+
+ To turn on the feature, go to the **Home** page of Kylin and click **Turn on Recommendation**.
+:::
+
+### **Background information**
+
+With the "Recommendation" feature enabled, recommendations will be automatically generated and temporarily recorded in the internal system. After you set up the generation rules, the system will only prompt the recommendations that meet the criteria. You can check the recommendations on the **Home** page.
+
+For more information about recommendation, see [Basic Concepts and Principles](basic_concept_actions.md).
+
+### **Operation steps**
+
+1. Log in to Kylin.
+
+ - System admin
+ - **Management** or **Admin** of the target project
+
+2. In the left navigation panel, click **Setting**.
+
+3. On the **Basic Settings** tab, go to the **Recommendation Preferences** section.
+
+4. Specify the following parameters for the rule. Then click **Save** to finish the rule setting.
+
+ 
+
+ -
+
+
+5. On the **Basic Settings** tab, go to the **Exclude Column Rule Setting** section to specify exclude columns.
+
+ If turn on this switch, columns can be added after searching the specified table through keywords, and the system will not generate optimization suggestions for exclude columns.
+
+ * Exclude columns is not suitable for fact table. In fact, if a table having exclude columns is chosen as fact table in a model, the excluding rules actually will do nothing for these columns.
+ * Exclude columns are used to handle slowly changing dimensions, especially when you want to use SCD type 1 for some columns and SCD type 2 for others.
+ * If you enable intelligent recommendation, the exclude columns will not be recommended to the optimization suggestions. This function applies to the scenario of AS-IS (SCD type 1) analysis.
+ * After the Settings take effect, the foreign keys in the fact table are recommended for optimization recommendations instead of the exclude columns, and the columns in the query table can be derived answer from the foreign keys after the index is built.
+ * If exclude columns are already used in the indexes, you need to manually delete these indexes dependent on these columns to implement AS-IS (SCD type 1).
+ * If you want to implement AS-WAS (SCD type 2) on other columns of lookup table, these columns can be added into index
+
+:::info
+
+ * By default，the 'Exclude Table' function is replaced by 'Exclude Column'，you can set `kylin.metadata.table-exclusion-enabled=true` to enable it in project settings.
+ * You can exclude all columns on a table to get the effect of exclude table.
+ * The former parameter of exclude table: `kylin.engine.build-excluded-table=true` is now abandon.
+:::
+
+
+
+-
+
diff --git a/website/docs/model/rec/sql_modeling.md b/website/docs/model/rec/sql_modeling.md
new file mode 100755
index 0000000000..0ea4e28d57
--- /dev/null
+++ b/website/docs/model/rec/sql_modeling.md
@@ -0,0 +1,138 @@
+---
+title: Imported SQL Modeling
+language: en
+sidebar_label: Imported SQL Modeling
+pagination_label: Imported SQL Modeling
+toc_min_heading_level: 2
+toc_max_heading_level: 6
+pagination_prev: null
+pagination_next: null
+keywords:
+ - recommendation
+draft: false
+last_update:
+ date: 09/02/2024
+---
+
+This chapter we will show you how to create model and index by SQL.
+
+### **Prerequisites**
+
+- [Load data source](../../datasource/intro.md). This article takes this [SSB sample dataset](../../quickstart/tutorial.md) as an example to build model and indexes.
+
+- Save the SQL statements as a *.txt* or *.sql* file. The following SQL statement is used in this article.
+
+
+
+ :::tip Tips
+ The file size should be no larger than 5 MB. If there are multiple SQL statements, separate them by semicolons (;).
+ :::
+- Create a project if you have not created any projects yet. For more details, refer to [Create project](../../operations/project-managing/project_management.md).
+- Turn on recommendation function in the project setting page, this has been introduced in the chapter [Recommendation](intro.md).
+
+### Create Model
+
+The following figure shows the entrance to SQL modeling. Please select **From SQL**, and the system will prompt you to upload the file. After the upload is completed, please click Next.
+
+
+
+In this example, we use the SQL statements as following.
+
+```sql
+-- valid sql
+select 1 from SSB.CUSTOMER;
+select C_CUSTKEY from SSB.CUSTOMER;
+
+-- invalid sql, T_PRICE_PER_ITEM looks like a computed column name
+select sum(T_PRICE_PER_ITEM) from SSB.P_LINEORDER;
+
+-- valid sql
+select
+ CUSTOMER.C_NATION,
+ CUSTOMER.C_CITY,
+ sum(P_LINEORDER.LO_SUPPLYCOST + 1),
+ sum(P_LINEORDER.LO_REVENUE)
+from
+ SSB.P_LINEORDER
+left join SSB.CUSTOMER on P_LINEORDER.LO_CUSTKEY = CUSTOMER.C_CUSTKEY
+left join SSB.PART on P_LINEORDER.LO_PARTKEY = PART.P_PARTKEY
+group by
+ CUSTOMER.C_NATION,
+ CUSTOMER.C_CITY;
+
+-- valid sql, but useless, will be ignored by the engine
+select 1;
+
+-- invalid sql, bail out
+select AAA from CCB.TABLE1;
+```
+
+
+After the SQL has been analyzed, you will see a screen similar to the one shown below. In this example, there are two invalid SQL statements. You can safely ignore these and proceed to the next step.
+
+
+
+
+
+The **Preview** dialog box displays the model created from the SQL input. You will see two proposed models, along with their respective computed columns, dimensions, and measures. Although the indexes for the models are also generated, they are not displayed here, as the focus is primarily on the model itself.
+
+
+ - **Model Name**: It is automatically generated and takes the format `AUTO_MODEL_FACT TABLE NAME_NUMBER`. You can also manually adjust the model name. It can be any combination of numbers, letters, and underscores (`_`). The maximum length of the model name is 127 characters. When the fact table name is too long, the system will automatically truncate the fact table name from the end to ensure that the automatically generated model name is less than or equal to 127 characters. When t [...]
+ - **Fact Table**: The fact table of model.
+ - **SQL**: The sql related to the model. Similar SQL statements will be merged when generating models.
+ - **Add Base Indexes**: Basic indexes include base aggregated indexes and base table indexes. Base aggregated indexes contain all model dimensions and measures, and base table indexes contain columns used in all model dimensions and measures. It's recommended to keep it selected to avoid frequent [query push-down](../../query/push_down.md) which may increase query response times. By default, base indexes will automatically update with model changes.
+
+
+
+If you click the **OK** button, these two models will be added to the system. Then you can go to model list page, you will see a page as follows.
+
+
+
+
+
+### Convert to Index
+
+In this section, we'll demonstrate another case of creating indexes using SQL. Before proceeding, we need to bring the model *AUTO_MODEL_CUSTOMER_1* online. Once the model is online, we can import the following SQL statement, which will result in the following outcome.
+
+```sq
+select C_CITY from SSB.CUSTOMER group by C_CITY;
+```
+
+You have two options:
+
++ **Create New Models**: This option will add a new model to the system based on the SQL input.
+
++ **Convert to Recommendations**: This option will provide recommendations on the existing model and propose new indexes based on the query patterns.
+
+
+
+
+
+
+
+After selecting **Convert to Recommendations** and accepting the proposed result on the **Preview** page, you should see a confirmation indicating success. This means the recommended indexes have been successfully generated.
+
+To verify this, navigate to the **Index Overview** page, where you will be able to view the newly created indexes. This step allows you to confirm that the system has applied the optimizations and recommendations to improve query performance based on your model's structure.
+
+
+
+
+
+
+
+As shown in the picture, there is a **Recommended Aggregate Index** whose content is *\[C_CICTY, COUNT_ALL\]* corresponding to our input sql statement. Besides, the base indexes has been updated, you can check on your system.
+
+
+
+
+
+
+
+
+### Create New Model
+
+When you select **Create New Models**, the system will generate a new model, such as *AUTO_MODEL_CUSTOMER_2*, with *SSB.CUSTOMER* as the fact table. However, we against doing this, as it may lead to the creation of numerous similar models. Managing a large number of models can become difficult and inefficient.
+
+
+
+
diff --git a/website/docs/modeling/images/snapshot/snapshot_1.png b/website/docs/model/snapshot/images/snapshot_1.png
similarity index 100%
rename from website/docs/modeling/images/snapshot/snapshot_1.png
rename to website/docs/model/snapshot/images/snapshot_1.png
diff --git a/website/docs/modeling/images/snapshot/snapshot_2.png b/website/docs/model/snapshot/images/snapshot_2.png
similarity index 100%
rename from website/docs/modeling/images/snapshot/snapshot_2.png
rename to website/docs/model/snapshot/images/snapshot_2.png
diff --git a/website/docs/modeling/images/snapshot/snapshot_3.png b/website/docs/model/snapshot/images/snapshot_3.png
similarity index 100%
rename from website/docs/modeling/images/snapshot/snapshot_3.png
rename to website/docs/model/snapshot/images/snapshot_3.png
diff --git a/website/docs/modeling/images/snapshot/snapshot_4.png b/website/docs/model/snapshot/images/snapshot_4.png
similarity index 100%
rename from website/docs/modeling/images/snapshot/snapshot_4.png
rename to website/docs/model/snapshot/images/snapshot_4.png
diff --git a/website/docs/modeling/images/snapshot/snapshot_5.png b/website/docs/model/snapshot/images/snapshot_5.png
similarity index 100%
rename from website/docs/modeling/images/snapshot/snapshot_5.png
rename to website/docs/model/snapshot/images/snapshot_5.png
diff --git a/website/docs/modeling/images/snapshot/snapshot_6.png b/website/docs/model/snapshot/images/snapshot_6.png
similarity index 100%
rename from website/docs/modeling/images/snapshot/snapshot_6.png
rename to website/docs/model/snapshot/images/snapshot_6.png
diff --git a/website/docs/modeling/images/snapshot/snapshot_7.png b/website/docs/model/snapshot/images/snapshot_7.png
similarity index 100%
rename from website/docs/modeling/images/snapshot/snapshot_7.png
rename to website/docs/model/snapshot/images/snapshot_7.png
diff --git a/website/docs/modeling/images/snapshot/snapshot_8.png b/website/docs/model/snapshot/images/snapshot_8.png
similarity index 100%
rename from website/docs/modeling/images/snapshot/snapshot_8.png
rename to website/docs/model/snapshot/images/snapshot_8.png
diff --git a/website/docs/modeling/images/snapshot/snapshot_9.png b/website/docs/model/snapshot/images/snapshot_9.png
similarity index 100%
rename from website/docs/modeling/images/snapshot/snapshot_9.png
rename to website/docs/model/snapshot/images/snapshot_9.png
diff --git a/website/docs/modeling/snapshot_management.md b/website/docs/model/snapshot/snapshot.md
similarity index 90%
rename from website/docs/modeling/snapshot_management.md
rename to website/docs/model/snapshot/snapshot.md
index 41fc64ee5e..a1e76d23ba 100644
--- a/website/docs/modeling/snapshot_management.md
+++ b/website/docs/model/snapshot/snapshot.md
@@ -1,8 +1,8 @@
---
-title: Dimension Table Snapshot
+title: Table Snapshot
language: en
-sidebar_label: Dimension Table Snapshot
-pagination_label: Dimension Table Snapshot
+sidebar_label: Table Snapshot
+pagination_label: Table Snapshot
toc_min_heading_level: 2
toc_max_heading_level: 6
pagination_prev: null
@@ -32,11 +32,11 @@ The snapshot management is disabled by default. You can enable it in advanced se
1. Navigate to **Settings -> Advanced Settings -> Snapshot Management**, turn on **Support Snapshot Management**.
- 
+ 
- 
+ 
- 
+ 
**Note**:
@@ -50,7 +50,7 @@ The snapshot management is disabled by default. You can enable it in advanced se
2. Navigate to **Data Asset -> Snapshot**, the snapshot list will be displayed, as shown below:
- 
+ 
**Fields Explanation:**
@@ -76,7 +76,7 @@ Above the snapshot lis are the operation buttons. Specific actions are listed be
- **Add Snapshots**：Select tables or databases to build snapshots. Click **Next** to set partition columns.
- 
+ 
Next, set the snapshot partition columns. Setting snapshot partition columns can improve the building speed by building partitions in parallel to a certain extent. By default, the system will build in a non-partitioned manner. Users can also set or detect partition columns , Also support specified partition value refresh.
> **Note**:
@@ -85,23 +85,23 @@ Above the snapshot lis are the operation buttons. Specific actions are listed be
>
> - Building in partitions can only increase the speed of the building task, and the result of the built snapshot is still full data.
- 
+ 
- **Refresh Snapshots**：Select snapshots to refresh.
- Full refresh: The refresh operation at this time builds the latest and full snapshot.
- Incremental refresh: For snapshots with source table partitions set, only the newly added partition value data will be refreshed (the built historical data will not be refreshed). It is recommended to select when the historical data is not updated.
- Custom partition value refresh: Specify single or multiple partition values for data refresh.
- 
+ 
- **Delete Snapshots**：Select snapshots to delete. It will discard all jobs related to the snapshots.
- 
+ 
- **Repair Snapshots**: When the source table has a structural change and is reloaded, the corresponding snapshot will change to "BROKEN" and can be repaired.
- 
+ 
According building tasks of snapshot，you can configure spark related configurations in `kylin.properties` to achieve more fine-grained control.（These configuration can be overridden at project level）
diff --git a/website/docs/modeling/real-time/custom-parser.md b/website/docs/model/streaming/custom-parser.md
similarity index 100%
rename from website/docs/modeling/real-time/custom-parser.md
rename to website/docs/model/streaming/custom-parser.md
diff --git a/website/docs/modeling/real-time/images/add.png b/website/docs/model/streaming/images/add.png
similarity index 100%
rename from website/docs/modeling/real-time/images/add.png
rename to website/docs/model/streaming/images/add.png
diff --git a/website/docs/modeling/real-time/images/custom_project_configuration_en.png b/website/docs/model/streaming/images/custom_project_configuration_en.png
similarity index 100%
rename from website/docs/modeling/real-time/images/custom_project_configuration_en.png
rename to website/docs/model/streaming/images/custom_project_configuration_en.png
diff --git a/website/docs/modeling/real-time/images/index_data_range_en.png b/website/docs/model/streaming/images/index_data_range_en.png
similarity index 100%
rename from website/docs/modeling/real-time/images/index_data_range_en.png
rename to website/docs/model/streaming/images/index_data_range_en.png
diff --git a/website/docs/modeling/real-time/images/load_tables_1_en.png b/website/docs/model/streaming/images/load_tables_1_en.png
similarity index 100%
rename from website/docs/modeling/real-time/images/load_tables_1_en.png
rename to website/docs/model/streaming/images/load_tables_1_en.png
diff --git a/website/docs/modeling/real-time/images/load_tables_2_en.png b/website/docs/model/streaming/images/load_tables_2_en.png
similarity index 100%
rename from website/docs/modeling/real-time/images/load_tables_2_en.png
rename to website/docs/model/streaming/images/load_tables_2_en.png
diff --git a/website/docs/modeling/real-time/images/more.png b/website/docs/model/streaming/images/more.png
similarity index 100%
rename from website/docs/modeling/real-time/images/more.png
rename to website/docs/model/streaming/images/more.png
diff --git a/website/docs/modeling/real-time/images/par_config.png b/website/docs/model/streaming/images/par_config.png
similarity index 100%
rename from website/docs/modeling/real-time/images/par_config.png
rename to website/docs/model/streaming/images/par_config.png
diff --git a/website/docs/modeling/real-time/images/real-time_add_hive_table.png b/website/docs/model/streaming/images/real-time_add_hive_table.png
similarity index 100%
rename from website/docs/modeling/real-time/images/real-time_add_hive_table.png
rename to website/docs/model/streaming/images/real-time_add_hive_table.png
diff --git a/website/docs/modeling/real-time/images/real-time_add_kafka_table.png b/website/docs/model/streaming/images/real-time_add_kafka_table.png
similarity index 100%
rename from website/docs/modeling/real-time/images/real-time_add_kafka_table.png
rename to website/docs/model/streaming/images/real-time_add_kafka_table.png
diff --git a/website/docs/modeling/real-time/images/real-time_add_model.png b/website/docs/model/streaming/images/real-time_add_model.png
similarity index 100%
rename from website/docs/modeling/real-time/images/real-time_add_model.png
rename to website/docs/model/streaming/images/real-time_add_model.png
diff --git a/website/docs/modeling/real-time/images/real-time_add_model_pic.png b/website/docs/model/streaming/images/real-time_add_model_pic.png
similarity index 100%
rename from website/docs/modeling/real-time/images/real-time_add_model_pic.png
rename to website/docs/model/streaming/images/real-time_add_model_pic.png
diff --git a/website/docs/modeling/real-time/images/real-time_arch.png b/website/docs/model/streaming/images/real-time_arch.png
similarity index 100%
rename from website/docs/modeling/real-time/images/real-time_arch.png
rename to website/docs/model/streaming/images/real-time_arch.png
diff --git a/website/docs/modeling/real-time/images/real-time_arch1.png b/website/docs/model/streaming/images/real-time_arch1.png
similarity index 100%
rename from website/docs/modeling/real-time/images/real-time_arch1.png
rename to website/docs/model/streaming/images/real-time_arch1.png
diff --git a/website/docs/modeling/real-time/images/real-time_associate_with_hive.png b/website/docs/model/streaming/images/real-time_associate_with_hive.png
similarity index 100%
rename from website/docs/modeling/real-time/images/real-time_associate_with_hive.png
rename to website/docs/model/streaming/images/real-time_associate_with_hive.png
diff --git a/website/docs/modeling/real-time/images/real-time_build_batch_index.png b/website/docs/model/streaming/images/real-time_build_batch_index.png
similarity index 100%
rename from website/docs/modeling/real-time/images/real-time_build_batch_index.png
rename to website/docs/model/streaming/images/real-time_build_batch_index.png
diff --git a/website/docs/modeling/real-time/images/real-time_build_index.png b/website/docs/model/streaming/images/real-time_build_index.png
similarity index 100%
rename from website/docs/modeling/real-time/images/real-time_build_index.png
rename to website/docs/model/streaming/images/real-time_build_index.png
diff --git a/website/docs/modeling/real-time/images/real-time_build_index_time_range.png b/website/docs/model/streaming/images/real-time_build_index_time_range.png
similarity index 100%
rename from website/docs/modeling/real-time/images/real-time_build_index_time_range.png
rename to website/docs/model/streaming/images/real-time_build_index_time_range.png
diff --git a/website/docs/modeling/real-time/images/real-time_build_realtime_index.png b/website/docs/model/streaming/images/real-time_build_realtime_index.png
similarity index 100%
rename from website/docs/modeling/real-time/images/real-time_build_realtime_index.png
rename to website/docs/model/streaming/images/real-time_build_realtime_index.png
diff --git a/website/docs/modeling/real-time/images/real-time_dimension_table_timed_refresh.png b/website/docs/model/streaming/images/real-time_dimension_table_timed_refresh.png
similarity index 100%
rename from website/docs/modeling/real-time/images/real-time_dimension_table_timed_refresh.png
rename to website/docs/model/streaming/images/real-time_dimension_table_timed_refresh.png
diff --git a/website/docs/modeling/real-time/images/real-time_finish_add_source.png b/website/docs/model/streaming/images/real-time_finish_add_source.png
similarity index 100%
rename from website/docs/modeling/real-time/images/real-time_finish_add_source.png
rename to website/docs/model/streaming/images/real-time_finish_add_source.png
diff --git a/website/docs/modeling/real-time/images/real-time_kafka_message_sample.png b/website/docs/model/streaming/images/real-time_kafka_message_sample.png
similarity index 100%
rename from website/docs/modeling/real-time/images/real-time_kafka_message_sample.png
rename to website/docs/model/streaming/images/real-time_kafka_message_sample.png
diff --git a/website/docs/modeling/real-time/images/real-time_monitor_batch_status.png b/website/docs/model/streaming/images/real-time_monitor_batch_status.png
similarity index 100%
rename from website/docs/modeling/real-time/images/real-time_monitor_batch_status.png
rename to website/docs/model/streaming/images/real-time_monitor_batch_status.png
diff --git a/website/docs/modeling/real-time/images/real-time_monitor_realtime_config.png b/website/docs/model/streaming/images/real-time_monitor_realtime_config.png
similarity index 100%
rename from website/docs/modeling/real-time/images/real-time_monitor_realtime_config.png
rename to website/docs/model/streaming/images/real-time_monitor_realtime_config.png
diff --git a/website/docs/modeling/real-time/images/real-time_monitor_realtime_diagnostics.png b/website/docs/model/streaming/images/real-time_monitor_realtime_diagnostics.png
similarity index 100%
rename from website/docs/modeling/real-time/images/real-time_monitor_realtime_diagnostics.png
rename to website/docs/model/streaming/images/real-time_monitor_realtime_diagnostics.png
diff --git a/website/docs/modeling/real-time/images/real-time_monitor_realtime_logs.png b/website/docs/model/streaming/images/real-time_monitor_realtime_logs.png
similarity index 100%
rename from website/docs/modeling/real-time/images/real-time_monitor_realtime_logs.png
rename to website/docs/model/streaming/images/real-time_monitor_realtime_logs.png
diff --git a/website/docs/modeling/real-time/images/real-time_monitor_realtime_logs_info.png b/website/docs/model/streaming/images/real-time_monitor_realtime_logs_info.png
similarity index 100%
rename from website/docs/modeling/real-time/images/real-time_monitor_realtime_logs_info.png
rename to website/docs/model/streaming/images/real-time_monitor_realtime_logs_info.png
diff --git a/website/docs/modeling/real-time/images/real-time_monitor_realtime_status.png b/website/docs/model/streaming/images/real-time_monitor_realtime_status.png
similarity index 100%
rename from website/docs/modeling/real-time/images/real-time_monitor_realtime_status.png
rename to website/docs/model/streaming/images/real-time_monitor_realtime_status.png
diff --git a/website/docs/modeling/real-time/images/real-time_query_batch_data.png b/website/docs/model/streaming/images/real-time_query_batch_data.png
similarity index 100%
rename from website/docs/modeling/real-time/images/real-time_query_batch_data.png
rename to website/docs/model/streaming/images/real-time_query_batch_data.png
diff --git a/website/docs/modeling/real-time/images/real-time_query_streaming_data.png b/website/docs/model/streaming/images/real-time_query_streaming_data.png
similarity index 100%
rename from website/docs/modeling/real-time/images/real-time_query_streaming_data.png
rename to website/docs/model/streaming/images/real-time_query_streaming_data.png
diff --git a/website/docs/modeling/real-time/images/real-time_topic.png b/website/docs/model/streaming/images/real-time_topic.png
similarity index 100%
rename from website/docs/modeling/real-time/images/real-time_topic.png
rename to website/docs/model/streaming/images/real-time_topic.png
diff --git a/website/docs/modeling/real-time/images/select_custom_parser.png b/website/docs/model/streaming/images/select_custom_parser.png
similarity index 100%
rename from website/docs/modeling/real-time/images/select_custom_parser.png
rename to website/docs/model/streaming/images/select_custom_parser.png
diff --git a/website/docs/modeling/real-time/intro.md b/website/docs/model/streaming/intro.md
similarity index 100%
rename from website/docs/modeling/real-time/intro.md
rename to website/docs/model/streaming/intro.md
diff --git a/website/docs/modeling/real-time/prerequisite.md b/website/docs/model/streaming/prerequisite.md
similarity index 100%
rename from website/docs/modeling/real-time/prerequisite.md
rename to website/docs/model/streaming/prerequisite.md
diff --git a/website/docs/modeling/real-time/real-time.md b/website/docs/model/streaming/real-time.md
similarity index 100%
rename from website/docs/modeling/real-time/real-time.md
rename to website/docs/model/streaming/real-time.md
diff --git a/website/docs/modeling/real-time/streaming_sdk/images/streaming_sdk_create_project.en.png b/website/docs/model/streaming/streaming_sdk/images/streaming_sdk_create_project.en.png
similarity index 100%
rename from website/docs/modeling/real-time/streaming_sdk/images/streaming_sdk_create_project.en.png
rename to website/docs/model/streaming/streaming_sdk/images/streaming_sdk_create_project.en.png
diff --git a/website/docs/modeling/real-time/streaming_sdk/images/streaming_sdk_create_table_csv_1.en.png b/website/docs/model/streaming/streaming_sdk/images/streaming_sdk_create_table_csv_1.en.png
similarity index 100%
rename from website/docs/modeling/real-time/streaming_sdk/images/streaming_sdk_create_table_csv_1.en.png
rename to website/docs/model/streaming/streaming_sdk/images/streaming_sdk_create_table_csv_1.en.png
diff --git a/website/docs/modeling/real-time/streaming_sdk/images/streaming_sdk_create_table_csv_2.en.png b/website/docs/model/streaming/streaming_sdk/images/streaming_sdk_create_table_csv_2.en.png
similarity index 100%
rename from website/docs/modeling/real-time/streaming_sdk/images/streaming_sdk_create_table_csv_2.en.png
rename to website/docs/model/streaming/streaming_sdk/images/streaming_sdk_create_table_csv_2.en.png
diff --git a/website/docs/modeling/real-time/streaming_sdk/images/streaming_sdk_create_table_csv_3.en.png b/website/docs/model/streaming/streaming_sdk/images/streaming_sdk_create_table_csv_3.en.png
similarity index 100%
rename from website/docs/modeling/real-time/streaming_sdk/images/streaming_sdk_create_table_csv_3.en.png
rename to website/docs/model/streaming/streaming_sdk/images/streaming_sdk_create_table_csv_3.en.png
diff --git a/website/docs/modeling/real-time/streaming_sdk/images/streaming_sdk_create_table_json_1.en.png b/website/docs/model/streaming/streaming_sdk/images/streaming_sdk_create_table_json_1.en.png
similarity index 100%
rename from website/docs/modeling/real-time/streaming_sdk/images/streaming_sdk_create_table_json_1.en.png
rename to website/docs/model/streaming/streaming_sdk/images/streaming_sdk_create_table_json_1.en.png
diff --git a/website/docs/modeling/real-time/streaming_sdk/images/streaming_sdk_create_table_json_2.en.png b/website/docs/model/streaming/streaming_sdk/images/streaming_sdk_create_table_json_2.en.png
similarity index 100%
rename from website/docs/modeling/real-time/streaming_sdk/images/streaming_sdk_create_table_json_2.en.png
rename to website/docs/model/streaming/streaming_sdk/images/streaming_sdk_create_table_json_2.en.png
diff --git a/website/docs/modeling/real-time/streaming_sdk/images/streaming_sdk_create_table_json_3.en.png b/website/docs/model/streaming/streaming_sdk/images/streaming_sdk_create_table_json_3.en.png
similarity index 100%
rename from website/docs/modeling/real-time/streaming_sdk/images/streaming_sdk_create_table_json_3.en.png
rename to website/docs/model/streaming/streaming_sdk/images/streaming_sdk_create_table_json_3.en.png
diff --git a/website/docs/modeling/real-time/streaming_sdk/images/streaming_sdk_lib.en.png b/website/docs/model/streaming/streaming_sdk/images/streaming_sdk_lib.en.png
similarity index 100%
rename from website/docs/modeling/real-time/streaming_sdk/images/streaming_sdk_lib.en.png
rename to website/docs/model/streaming/streaming_sdk/images/streaming_sdk_lib.en.png
diff --git a/website/docs/modeling/real-time/streaming_sdk/images/streaming_sdk_metainf.en.png b/website/docs/model/streaming/streaming_sdk/images/streaming_sdk_metainf.en.png
similarity index 100%
rename from website/docs/modeling/real-time/streaming_sdk/images/streaming_sdk_metainf.en.png
rename to website/docs/model/streaming/streaming_sdk/images/streaming_sdk_metainf.en.png
diff --git a/website/docs/modeling/real-time/streaming_sdk/images/streaming_sdk_package.en.png b/website/docs/model/streaming/streaming_sdk/images/streaming_sdk_package.en.png
similarity index 100%
rename from website/docs/modeling/real-time/streaming_sdk/images/streaming_sdk_package.en.png
rename to website/docs/model/streaming/streaming_sdk/images/streaming_sdk_package.en.png
diff --git a/website/docs/modeling/real-time/streaming_sdk/streaming_sdk.md b/website/docs/model/streaming/streaming_sdk/streaming_sdk.md
similarity index 100%
rename from website/docs/modeling/real-time/streaming_sdk/streaming_sdk.md
rename to website/docs/model/streaming/streaming_sdk/streaming_sdk.md
diff --git a/website/docs/modeling/auto_modeling/data_modeling_by_SQL.md b/website/docs/modeling/auto_modeling/data_modeling_by_SQL.md
deleted file mode 100755
index cec3418586..0000000000
--- a/website/docs/modeling/auto_modeling/data_modeling_by_SQL.md
+++ /dev/null
@@ -1,90 +0,0 @@
----
-title: Auto Modeling with SQL
-language: en
-sidebar_label: Auto Modeling with SQL
-pagination_label: Auto Modeling with SQL
-toc_min_heading_level: 2
-toc_max_heading_level: 6
-pagination_prev: modeling/model_design/intro
-pagination_next: null
-keywords:
- - recommendation
-draft: false
-last_update:
- date: 09/02/2024
----
-
-Kylin model consists of tables and their join relations, as well as defined dimensions and measures. After you upload SQL statements, Kylin automatically parses dimensions, measures and other critical information, and guides you through the modeling process. You do not need profound knowledge to design a model that satisfies your needs, which greatly shortens the model development cycle.
-
-### **Prerequisites**
-
-- [Load data source](../../datasource/intro.md). This article takes this [SSB sample dataset](../../quickstart/tutorial.md) as an example to build model and indexes.
-
-- Save the SQL statements as a TXT or SQL file. The following SQL statement is used in this article.
-
- ```sql
- SELECT
- CUSTOMER.C_NATION, CUSTOMER.C_CITY, CUSTOMER.C_REGION, DATES.D_YEAR, DATES.D_DATE, PART.P_BRAND
- ,sum(P_LINEORDER.LO_SUPPLYCOST), sum(P_LINEORDER.LO_REVENUE),count(1)
- FROM
- "SSB"."P_LINEORDER" as "P_LINEORDER"
- LEFT JOIN "SSB"."CUSTOMER" as "CUSTOMER" ON "P_LINEORDER"."LO_CUSTKEY"="CUSTOMER"."C_CUSTKEY"
- LEFT JOIN "SSB"."DATES" as "DATES" ON "P_LINEORDER"."LO_ORDERDATE"="DATES"."D_DATEKEY"
- LEFT JOIN "SSB"."PART" as "PART" ON "P_LINEORDER"."LO_PARTKEY"="PART"."P_PARTKEY"
- LEFT JOIN "SSB"."SUPPLIER" as "SUPPLIER" ON "P_LINEORDER"."LO_SUPPKEY"="SUPPLIER"."S_SUPPKEY"
- group by CUSTOMER.C_NATION, CUSTOMER.C_CITY, CUSTOMER.C_REGION, DATES.D_YEAR, DATES.D_DATE, PART.P_BRAND;
- ```
-
- > [!NOTE]
- >
- > The file size should be no larger than 5 MB. If there are multiple SQL statements, separate them by semicolons (;).
-
-
-### Operation steps
-
-1. Log in to Kylin as either of the following roles:
- - System admin
- - **Admin** of the target project
-
-2. Enable Intelligent Recommendation for the target project. Create a project if you have not created any projects yet. For more information, see [Create project](../../operations/project-managing/project_management.md).
- 1. On the **Home** page, click **Turn on Recommendation**.
-
- 
-
- 2. Click **Turn on** in the confirmation dialog box.
-
-3. In the left navigation panel, click **Data Assets** > **Data Source**.
-
-4. Select **+ Model** > **From SQL**.
-
- 
-
-5. In the pop-up dialog box, click **Upload**. Select and upload the SQL file, and then click **Next**.
-
-6. In the **Import SQL** dialog box, check if the SQL queries are correctly parsed. Then click **Next**.
-
- 
-
- > [!NOTE]
- >
- > - Kylin merges similar SQL statements. You can select, edit or delete the SQL statements to be loaded.
- >
- > - Kylin will suggest converting existing models that match certain SQL queries into recommendations, to help control model numbers.
-
-7. In the **Preview** dialog box, check the dimensions, measures, computed columns and other model information. You can also modify the following configurations.
-
- 
-
- - **Model Name**: It is automatically generated and takes the format `AUTO_MODEL_FACT TABLE NAME_NUMBER`. You can also manually adjust the model name. It can be any combination of numbers, letters, and underscores (`_`). The maximum length of the model name is 127 characters. When the fact table name is too long, the system will automatically truncate the fact table name from the end to ensure that the automatically generated model name is less than or equal to 127 characters. When t [...]
- - **Add Base Indexes**: Basic indexes include base aggregated indexes and base table indexes. Base aggregated indexes contain all model dimensions and measures, and base table indexes contain columns used in all model dimensions and measures.
-
- It's recommended to keep it selected to avoid frequent [query push_down](../../query/push_down.md) which may increase query response times. By default, base indexes will automatically update with model changes.
-
-8. Click **OK**. Kylin will automatically start to design a model and indexes.
-
-### Next steps
-
-For the indexes just created, you will need to [load data](../load_data/build_index.md), that is, to pre-compute the data.
-
-After the index building, these indexes can greatly help to accelerate query response times.
-
diff --git a/website/docs/modeling/auto_modeling/images/acceleration_rule.en.png b/website/docs/modeling/auto_modeling/images/acceleration_rule.en.png
deleted file mode 100644
index 8f8a148c6e..0000000000
Binary files a/website/docs/modeling/auto_modeling/images/acceleration_rule.en.png and /dev/null differ
diff --git a/website/docs/modeling/auto_modeling/images/ai_augment_principle.png b/website/docs/modeling/auto_modeling/images/ai_augment_principle.png
deleted file mode 100644
index 48c87199d5..0000000000
Binary files a/website/docs/modeling/auto_modeling/images/ai_augment_principle.png and /dev/null differ
diff --git a/website/docs/modeling/auto_modeling/images/build.png b/website/docs/modeling/auto_modeling/images/build.png
deleted file mode 100644
index c4f7f52d6a..0000000000
Binary files a/website/docs/modeling/auto_modeling/images/build.png and /dev/null differ
diff --git a/website/docs/modeling/auto_modeling/images/build_index_en.png b/website/docs/modeling/auto_modeling/images/build_index_en.png
deleted file mode 100644
index df64a268bc..0000000000
Binary files a/website/docs/modeling/auto_modeling/images/build_index_en.png and /dev/null differ
diff --git a/website/docs/modeling/auto_modeling/images/formulas.en.png b/website/docs/modeling/auto_modeling/images/formulas.en.png
deleted file mode 100644
index 7422218117..0000000000
Binary files a/website/docs/modeling/auto_modeling/images/formulas.en.png and /dev/null differ
diff --git a/website/docs/modeling/auto_modeling/images/fromsql.png b/website/docs/modeling/auto_modeling/images/fromsql.png
deleted file mode 100644
index 5f6a6ee7be..0000000000
Binary files a/website/docs/modeling/auto_modeling/images/fromsql.png and /dev/null differ
diff --git a/website/docs/modeling/auto_modeling/images/import_sql_en.png b/website/docs/modeling/auto_modeling/images/import_sql_en.png
deleted file mode 100644
index 23b7147261..0000000000
Binary files a/website/docs/modeling/auto_modeling/images/import_sql_en.png and /dev/null differ
diff --git a/website/docs/modeling/auto_modeling/images/main_flow.en.png b/website/docs/modeling/auto_modeling/images/main_flow.en.png
deleted file mode 100644
index 33948fc6ff..0000000000
Binary files a/website/docs/modeling/auto_modeling/images/main_flow.en.png and /dev/null differ
diff --git a/website/docs/modeling/auto_modeling/images/preview_en.png b/website/docs/modeling/auto_modeling/images/preview_en.png
deleted file mode 100644
index 9acb0e2c1b..0000000000
Binary files a/website/docs/modeling/auto_modeling/images/preview_en.png and /dev/null differ
diff --git a/website/docs/modeling/auto_modeling/images/rule_setting.en.png b/website/docs/modeling/auto_modeling/images/rule_setting.en.png
deleted file mode 100644
index 6538741e6c..0000000000
Binary files a/website/docs/modeling/auto_modeling/images/rule_setting.en.png and /dev/null differ
diff --git a/website/docs/modeling/auto_modeling/images/turn_on_rec.png b/website/docs/modeling/auto_modeling/images/turn_on_rec.png
deleted file mode 100644
index f1c7e19fe1..0000000000
Binary files a/website/docs/modeling/auto_modeling/images/turn_on_rec.png and /dev/null differ
diff --git a/website/docs/modeling/auto_modeling/images/view.png b/website/docs/modeling/auto_modeling/images/view.png
deleted file mode 100644
index 237e1f349a..0000000000
Binary files a/website/docs/modeling/auto_modeling/images/view.png and /dev/null differ
diff --git a/website/docs/modeling/auto_modeling/images/wizard.png b/website/docs/modeling/auto_modeling/images/wizard.png
deleted file mode 100644
index fed52d59fd..0000000000
Binary files a/website/docs/modeling/auto_modeling/images/wizard.png and /dev/null differ
diff --git a/website/docs/modeling/auto_modeling/intro.md b/website/docs/modeling/auto_modeling/intro.md
deleted file mode 100644
index b2dd9fa3fc..0000000000
--- a/website/docs/modeling/auto_modeling/intro.md
+++ /dev/null
@@ -1,29 +0,0 @@
----
-title: Auto Modeling
-language: en
-sidebar_label: Auto Modeling
-pagination_label: Auto Modeling
-toc_min_heading_level: 2
-toc_max_heading_level: 6
-pagination_prev: null
-pagination_next: null
-keywords:
- - recommendation
-draft: false
-last_update:
- date: 09/02/2024
----
-
-## Auto Modeling (Recommended)
-
-Conventional modeling relies heavily on the data modelers' expertise. Powered by the recommendation engine, Kylin democratizes big data analytics with intelligent model and index building, as well as model optimization recommendations based on query history.
-
-Kylin recommendation engine offers:
-
-- [Intelligent Modeling](data_modeling_by_SQL.md): Kylin recommendation engine will collect dimensions, measures and other critical information from your SQL statements, and then guide you through the modeling process. You do not need profound knowledge to build models that satisfy your needs, so you can focus more on your business.
-
-- [Intelligent Index Optimization](optimize_index/intro.md): Indexes can help accelerate data queries. Kylin recommendation engine will generate recommendations based on business data characteristics and user queries to help precisely and efficiently optimize indexes. Kylin will also pre-compute data by loading it to indexes to quickly reduce query response times.
-
-### **See also**
-
-[Intelligent Recommendation Principles](basic_concept_actions.md)
diff --git a/website/docs/modeling/auto_modeling/rule_setting.md b/website/docs/modeling/auto_modeling/rule_setting.md
deleted file mode 100644
index e2d2920381..0000000000
--- a/website/docs/modeling/auto_modeling/rule_setting.md
+++ /dev/null
@@ -1,99 +0,0 @@
----
-title: Set Recommendation Rules
-language: en
-sidebar_label: Set Recommendation Rules
-pagination_label: Set Recommendation Rules
-toc_min_heading_level: 2
-toc_max_heading_level: 6
-pagination_prev: null
-pagination_next: null
-keywords:
- - recommendation
- - rule
-draft: false
-last_update:
- date: 09/02/2024
----
-
-Kylin optimizes query performance with index or model recommendations based on query history, data characteristics, SQL modeling, etc. The [Recommendation](intro.md) feature can simplify model building and accelerate queries. This article covers how to configure the setting for the recommendations.
-
-### **Prerequisites**
-
-Please make sure that the **Recommendation** feature is turned on.
-
-:::info
-
- To turn on the feature, go to the **Home** page of Kylin and click **Turn on Recommendation**.
-:::
-
-### **Background information**
-
-With the "Recommendation" feature enabled, recommendations will be automatically generated and temporarily recorded in the internal system. After you set up the generation rules, the system will only prompt the recommendations that meet the criteria. You can check the recommendations on the **Home** page.
-
-For more information about recommendation, see [Basic Concepts and Principles](basic_concept_actions.md).
-
-### **Operation steps**
-
-1. Log in to Kylin.
-
- - System admin
- - **Management** or **Admin** of the target project
-
-2. In the left navigation panel, click **Setting**.
-
-3. On the **Basic Settings** tab, go to the **Recommendation Preferences** section.
-
-4. Specify the following parameters for the rule. Then click **Save** to finish the rule setting.
-
- 
-
- - **User Rules:** Turn on the toggle switch if you only want to generate recommendations for selected users and/or user groups. Then you can select users and/or user groups in the following field(s).
-
- - **Duration Rule:** Generate recommendations for queries within certain duration. Then you can customize the query duration. Default: 5 to 3600 second(s).
-
-
- - **Hit Rules**: Set the parameters below to filter out indexes that are frequently hit by queries, that is, the indexes of high value.
- For example, if the **Time Frame** is set to 2 and **Number of Hits** is set to 30, the system will only prompt recommendations with at least 30 query hits in the past 2 days.
- - **Time Frame**: Set the number of days to be calculated for the query hits, starting from today, default: 2 days.
- - **Number of Hits**: Set the number of query hits (default: 30 hits). This hit count can later be modified based on the recommendations.
-
- - **Limit of Recommendations for Adding Index**: Set how many recommendations will be prompted and the push frequency.
-
- :::info
-
- This parameter is only valid for recommendations based on query history.
- - Number of recommendations: Set the number of recommendations that will be pushed. Recommendations will be ranked based on the query hit rates (from highest to lowest); the default value is 20, indicating only the top 20 recommendations will be prompted.
- - **Recommendation Frequency**: Set the frequency (in days) to push the recommendations, default: 2 days.
- :::
-5. On the **Basic Settings** tab, go to the **Exclude Column Rule Setting** section to specify exclude columns.
-
- If turn on this switch, columns can be added after searching the specified table through keywords, and the system will not generate optimization suggestions for exclude columns.
-
- * Exclude columns is not suitable for fact table. In fact, if a table having exclude columns is chosen as fact table in a model, the excluding rules actually will do nothing for these columns.
- * Exclude columns are used to handle slowly changing dimensions, especially when you want to use SCD type 1 for some columns and SCD type 2 for others.
- * If you enable intelligent recommendation, the exclude columns will not be recommended to the optimization suggestions. This function applies to the scenario of AS-IS (SCD type 1) analysis.
- * After the Settings take effect, the foreign keys in the fact table are recommended for optimization recommendations instead of the exclude columns, and the columns in the query table can be derived answer from the foreign keys after the index is built.
- * If exclude columns are already used in the indexes, you need to manually delete these indexes dependent on these columns to implement AS-IS (SCD type 1).
- * If you want to implement AS-WAS (SCD type 2) on other columns of lookup table, these columns can be added into index
-
-:::info
-
- * By default，the 'Exclude Table' function is replaced by 'Exclude Column'，you can set `kylin.metadata.table-exclusion-enabled=true` to enable it in project settings.
- * You can exclude all columns on a table to get the effect of exclude table.
- * The former parameter of exclude table: `kylin.engine.build-excluded-table=true` is now abandon.
-:::
-
-:::info Exclude Column
- * For join relation, in snowflake model, A join B(on a1 = b1) and B join C (on b2 = c2), the foreign key b2 cannot be a exclude column, otherwise the join chain will be broken when proposing recommendations.
- * The design of computed columns indicates that precomputation is required, while the design of exclude columns indicates that precomputation is not required. So these two are incompatible with each other. Therefore, it is not recommended to define the computed columns contains exclude columns in principle. In particular, computed columns and measures that dependent on exclude columns may result in incorrect data if they hit the index.
- * Computed columns dependent on excluded columns are not reused for the intelligent recommendation. Moreover, with the setting of `kylin.metadata.only-reuse-user-defined-computed column=true`(by default false), only the customized computed columns will be used for recommendations, that is to say, no more computed columns started with `CC_AUTO` will be created in the model.
- * For queries, by default indexes answer the query with a higher priority than using the foreign key and snapshot. After the exclude column is enabled and setting `kylin.query.snapshot-preferred-for-table-exclusion=true`, the system preferentially selects the foreign key index to runtime join snapshot to response the query. When the '**Exclude Column Rule Setting**' is enabled, this is the default behavior. However, by setting it to false, index independent on exclude columns will be pr [...]
-
-:::
-
-### **FAQ**
-
-- Question: At which level will the **Recommendations Preferences** be effective?
-
- Answer: At the project level. To exclude certain tables at the model level, that is, this table will not be calculated during the pre-computation and indexing, please deselect the **Precompute Join Relationships** checkbox in the **Add Join Relationship** dialog box.
-
diff --git a/website/docs/modeling/images/add_dimention.gif b/website/docs/modeling/images/add_dimention.gif
deleted file mode 100644
index 5ed9e71cab..0000000000
Binary files a/website/docs/modeling/images/add_dimention.gif and /dev/null differ
diff --git a/website/docs/modeling/images/add_join_relations.png b/website/docs/modeling/images/add_join_relations.png
deleted file mode 100644
index 93fc636fd2..0000000000
Binary files a/website/docs/modeling/images/add_join_relations.png and /dev/null differ
diff --git a/website/docs/modeling/images/add_measure.gif b/website/docs/modeling/images/add_measure.gif
deleted file mode 100644
index 1be35075e8..0000000000
Binary files a/website/docs/modeling/images/add_measure.gif and /dev/null differ
diff --git a/website/docs/modeling/images/add_tables.png b/website/docs/modeling/images/add_tables.png
deleted file mode 100644
index 99ec4133fe..0000000000
Binary files a/website/docs/modeling/images/add_tables.png and /dev/null differ
diff --git a/website/docs/modeling/images/create_join_relations.gif b/website/docs/modeling/images/create_join_relations.gif
deleted file mode 100644
index 5ed9e71cab..0000000000
Binary files a/website/docs/modeling/images/create_join_relations.gif and /dev/null differ
diff --git a/website/docs/modeling/images/reduceio.png b/website/docs/modeling/images/reduceio.png
deleted file mode 100644
index 2eb123fb1e..0000000000
Binary files a/website/docs/modeling/images/reduceio.png and /dev/null differ
diff --git a/website/docs/modeling/images/responsetime_datavolume.png b/website/docs/modeling/images/responsetime_datavolume.png
deleted file mode 100644
index 78086afd35..0000000000
Binary files a/website/docs/modeling/images/responsetime_datavolume.png and /dev/null differ
diff --git a/website/docs/modeling/images/save_load_method.png b/website/docs/modeling/images/save_load_method.png
deleted file mode 100644
index d50d3c6821..0000000000
Binary files a/website/docs/modeling/images/save_load_method.png and /dev/null differ
diff --git a/website/docs/modeling/images/star_model_created.png b/website/docs/modeling/images/star_model_created.png
deleted file mode 100644
index 94bf255e82..0000000000
Binary files a/website/docs/modeling/images/star_model_created.png and /dev/null differ
diff --git a/website/docs/modeling/images/switch_to_fact_table.png b/website/docs/modeling/images/switch_to_fact_table.png
deleted file mode 100644
index 70d0f4298c..0000000000
Binary files a/website/docs/modeling/images/switch_to_fact_table.png and /dev/null differ
diff --git a/website/docs/modeling/images/time_format.png b/website/docs/modeling/images/time_format.png
deleted file mode 100644
index 690c8c95fc..0000000000
Binary files a/website/docs/modeling/images/time_format.png and /dev/null differ
diff --git a/website/docs/modeling/intro.md b/website/docs/modeling/intro.md
deleted file mode 100644
index 01898b1f9e..0000000000
--- a/website/docs/modeling/intro.md
+++ /dev/null
@@ -1,63 +0,0 @@
----
-title: Model
-language: en
-sidebar_label: Model
-pagination_label: Model
-toc_min_heading_level: 2
-toc_max_heading_level: 6
-pagination_prev: null
-pagination_next: null
-keywords:
- - model
-draft: false
-last_update:
- date: 08/12/2022
----
-
-This chapter introduces the core concepts of Kylin: model and index design basics, advanced measures and load data.
-
-Kylin model consists of multiple tables and their join relations. Kylin follows multidimensional modeling theory to build star or snowflake schemas based on tables. Kylin also leverages pre-computation technique and will reuse the pre-computed results to answer queries, so there's no need to traverse all data when there is a query, thus achieving sub-second query times on PB-level data.
-
-### Challenges when dealing with massive data
-
-When analyzing massive data, there are some techniques to speed up computing and storage, but they cannot change the time complexity of query, that is, query latency and data volume are linearly dependent.
-
-If it takes 1 minute to query 100 million entries of data records, querying 10 billion data entries will take about 1 hour and 40 minutes. When companies want to analyze all business data piled up over the years or to add complexity to query, say, with more dimensions, queries will be running extremely slow or even time out.
-
-### Accelerate query with Kylin pre-computation
-
-Kylin leverages pre-computation to avoid the computing pressure brought by the growing data volume. That is, Kylin will precompute the combinations of defined model dimensions and then store the aggregated results as indexes to shorten query latency. In addition, Kylin uses parallel computing and columnar storage techniques to improve computing and storage speed.
-
-With pre-computation, the number of indexes will be determined by the dimension cardinality only, and will no longer undergo exponential growth as data volume increases. Taking the data analysis of online transactions as an example, with Kylin pre-computation, even if the volume of transaction data increases by 10 times, the query speed against the same analytical dimensions changes little. The computing time complexity can be kept at O(1), helping enterprises to analyze data more efficiently.
-
-
-
-### How to design model and indexes
-
-#### Manual modeling
-
-In addition to intelligent modeling, Kylin also supports users to design their own models and indexes based on their business needs. Kylin provides step-by-step guidance on how to complete basic model settings, including dimensions, measures, join relationships, and indexes. For details, see [Manual modeling](manual_modeling.md).
-
-#### Advanced model design
-
-Kylin offers various advanced features around models and indexes to help users quickly dig out the most valuable data. These features include:
-
-- Accelerated model design: Kylin offers built-in [advanced measures](model_design/measure_design/intro.md) like count distinct and Top N to speed up modeling.
-
-For more information, see [Advanced model design](model_design/advance_guide/intro.md).
-
-### Basic concepts
-
-Kylin follows multidimensional modeling theory and decomposes complex concepts into specific functional modules to make modeling easier. Below are some of the basic concepts used in Kylin:
-
-- Dimension: Dimension is a perspective of viewing data, which can be used to describe object attributes or characteristics, for example, product category.
-
-- Measure: Measure is an aggregated sum, which is usually a continuous value, for example, product sales.
-
-- Model: Model consists of multiple tables and their join relations, as well as defined dimensions and measures.
-
-- Pre-computation: Pre-computation is the process of aggregating data based on model dimension combinations and of storing the results as indexes to accelerate data query.
-
-- Index: Index is used to accelerate data query. Indexes are divided into:
- - Aggregate Index: Aggregate index is an aggregated combination of multiple dimensions and measures, and can be used to answer aggregate queries such as total sales for a given year.
- - Table Index: Table index is a multi-level index in a wide table and can be used to answer detailed queries such as the last 100 transactions of a certain user.
diff --git a/website/docs/modeling/load_data/build_index.md b/website/docs/modeling/load_data/build_index.md
deleted file mode 100755
index 9f48fd2d50..0000000000
--- a/website/docs/modeling/load_data/build_index.md
+++ /dev/null
@@ -1,47 +0,0 @@
----
-title: Build Index
-language: en
-sidebar_label: Build Index
-pagination_label: Build Index
-toc_min_heading_level: 2
-toc_max_heading_level: 6
-pagination_prev: null
-pagination_next: null
-keywords:
- - build index
-draft: false
-last_update:
- date: 08/19/2022
----
-
-As the business scenario changes, some of the indexes in the model need to be retained only in latest months for saving building and storage costs. Therefore, Kylin provides a more flexible way to build indexes since the 5.0 released.
-
-
-### Build Index
-
-In the **Index Overview** tab, we can see the index list and some basic information. In the **Index List**, we can filter some indexes by the keyword or ids and then only build them in selected segments. For example, some new columns are added in the source table because of the business demands. Therefore, we need to add some new indexes for those columns in the latest one month for analysis and cost saving. As shown in the figure below, we can select all the new and NO BUILD indexes, an [...]
-
-
-
-After that, please select the segment with the latest month and click the Build Index button to generate the building job. If you want to build the segments concurrently to improve the efficiency, you can also check the **Generate multiple segments in parallel** box. Then, the system will generate multiple jobs according to the number of selected segments.
-
-
-
-### Delete Index
-
-Similar to the building index, you can also delete some indexes in selected segments. For example, deleting some low frequent usage indexes in last year. As shown below, we can choose some of the indexes and click the **Delete** button to choose delete from all segments or parts of them.
-
-Note: If the indexes are deleted from segments, it may influence the query performance because some of query may route to the pushdown engine due to the lack of index.
-
-
-
-### Build All Index
-
-To support more flexible index building, it may expect that different indexes will be included in different segments. In order to ensure the stable query performance, we recommend you build all index among all segments after a period of time. Therefore, if the index is incomplete, we can quickly build all indexes by clicking the icon after the index data range.
-
-
-
-As shown below, all the segments with incomplete indexes will be shown after clicked the icon. Then, you can select all the segments and click **Build Index** to ensure segments with all indexes.
-
-
-
diff --git a/website/docs/modeling/load_data/by_date.md b/website/docs/modeling/load_data/by_date.md
deleted file mode 100755
index d300cbd977..0000000000
--- a/website/docs/modeling/load_data/by_date.md
+++ /dev/null
@@ -1,68 +0,0 @@
----
-title: Incremental Load
-language: en
-sidebar_label: Incremental Load
-pagination_label: Incremental Load
-toc_min_heading_level: 2
-toc_max_heading_level: 6
-pagination_prev: null
-pagination_next: null
-keywords:
- - incremental load
-draft: false
-last_update:
- date: 08/19/2022
----
-
-As your business data grows over time, you can choose to load data by date / time. The main contents are as follows:
-
-### <span id="expert">Load by Date/Time</span>
-
-If you have set a time partition column for your model, you can load data by date / time each time on the **Data Asset -> Model** page.
-
-1. **First Load**
- Open Kylin web UI and access the project. Select the model that needs to load in the model list page.
-
- **Step 1:** Click the **Build Index** button.
-
- 
-
- **Step 2:** Select the load range in the pop-up window and click the **Incremental Load** button. This action will trigger the job of **Load Data**.
-
- > **Note**:
- >
- > - You can click the icon on the right side of the load range to automatically detect the latest available range. When your mouse is hovering over the Icon **Detect available range** is displayed.
- > - When you load historical data at the first time and the data volume is too large, it may lead to a long loading time. Please be sure to set the load range according to your data volume, model complexity, and available resources.
-
- 
-
-
-
-### Known limitations
-The start time range of Segment should greater than `1970-01-02`.
-
-
- **Step 3:** After submission, go to the **Monitor -> Job** page, a list of running jobs will be displayed. The first job is the job we just submitted and **Data Range** is displayed as the selected load range in step 2.
-
- **Step 4:** When all steps are complete, the status of the job will become **Finished**. You can view the details in the model list. There is a Segment in the **Segment** tag and it is marked the **Start Time** and **End Time**.
-
- 
-
-2. **Incremental Load**
-
- After the first segment is built, we can build more segments incrementally to accommodate new data. The time range of two segments cannot overlap.
-
- The steps for incremental load are consistent with the steps described above. Click the **Build Index** button then select the load range in the pop-up window. To ensure continuity, a new segment always starts from the end of the last segment.
-
- When load completes, go to the model detail page and check there are two segments under the model.
-
- 
-
-3. **Add Segment**
-
- Besides the above methods, you can also increase the model data range by adding a new segment. Click the **+ Segment** button under segment list. Then click the **Incremental Build** button in the pop-up window. After that, a building job will be submitted and the third segment will be shown when the job is finished.
-
- Note: If you want to use this function, you need to enable **Creating Reserved Segments**. Please refer to [Project Settings](../../operations/project-managing/project_settings.md) for more information.
-
- 
- 
diff --git a/website/docs/modeling/load_data/full_build.md b/website/docs/modeling/load_data/full_build.md
deleted file mode 100644
index ffc3339d1f..0000000000
--- a/website/docs/modeling/load_data/full_build.md
+++ /dev/null
@@ -1,41 +0,0 @@
----
-title: Full Load
-language: en
-sidebar_label: Full Load
-pagination_label: Full Load
-toc_min_heading_level: 2
-toc_max_heading_level: 6
-pagination_prev: null
-pagination_next: null
-keywords:
- - full load
-draft: false
-last_update:
- date: 08/19/2022
----
-
-If you want to load all the data in the source table, you can choose full load. The main contents are as follows:
-
-### <span id="expert">Full Load</span>
-
-If you do not set a time partition column for your model, it will be full load each time.
-
-You cannot merge segments in a full load type model since there should be only one segment.
-
-Here we will introduce how to do a full load in the Web UI:
-
-1. Select the model that needs the full load in the model list. Click the **Build Index** button.
-
- 
-
-2. You will be prompted to load all data, including the data already loaded in the model.
-
- 
-
- > **Note**: If you load data for a model for the first time, the storage size in the above prompt will be 0.00 KB because the model has not been loaded (there is no data in the model).
-
-3. After that, you can view the build index job via the **Monitor -> Job** page.
-
-4. When the data is loaded, you can view the details in the model list. There is only one Segment in the **Segment** tag, and it is marked as full load.
-
- 
diff --git a/website/docs/modeling/load_data/images/build_empty_index_en.png b/website/docs/modeling/load_data/images/build_empty_index_en.png
deleted file mode 100644
index 1afc423fb3..0000000000
Binary files a/website/docs/modeling/load_data/images/build_empty_index_en.png and /dev/null differ
diff --git a/website/docs/modeling/load_data/images/build_summary.png b/website/docs/modeling/load_data/images/build_summary.png
deleted file mode 100644
index 2130345de0..0000000000
Binary files a/website/docs/modeling/load_data/images/build_summary.png and /dev/null differ
diff --git a/website/docs/modeling/load_data/images/customize_build_save_model.png b/website/docs/modeling/load_data/images/customize_build_save_model.png
deleted file mode 100644
index 0506f400ce..0000000000
Binary files a/website/docs/modeling/load_data/images/customize_build_save_model.png and /dev/null differ
diff --git a/website/docs/modeling/load_data/images/customize_build_save_model_partition.png b/website/docs/modeling/load_data/images/customize_build_save_model_partition.png
deleted file mode 100644
index decef08704..0000000000
Binary files a/website/docs/modeling/load_data/images/customize_build_save_model_partition.png and /dev/null differ
diff --git a/website/docs/modeling/load_data/images/full_build.png b/website/docs/modeling/load_data/images/full_build.png
deleted file mode 100644
index 518764c362..0000000000
Binary files a/website/docs/modeling/load_data/images/full_build.png and /dev/null differ
diff --git a/website/docs/modeling/load_data/images/full_build_cube.png b/website/docs/modeling/load_data/images/full_build_cube.png
deleted file mode 100644
index 2f19ee8621..0000000000
Binary files a/website/docs/modeling/load_data/images/full_build_cube.png and /dev/null differ
diff --git a/website/docs/modeling/load_data/images/full_build_segment_info.png b/website/docs/modeling/load_data/images/full_build_segment_info.png
deleted file mode 100644
index 2888af1daf..0000000000
Binary files a/website/docs/modeling/load_data/images/full_build_segment_info.png and /dev/null differ
diff --git a/website/docs/modeling/load_data/images/segment_2.png b/website/docs/modeling/load_data/images/segment_2.png
deleted file mode 100644
index be65246ae4..0000000000
Binary files a/website/docs/modeling/load_data/images/segment_2.png and /dev/null differ
diff --git a/website/docs/modeling/load_data/images/streaming/g.png b/website/docs/modeling/load_data/images/streaming/g.png
deleted file mode 100644
index c049b4e885..0000000000
Binary files a/website/docs/modeling/load_data/images/streaming/g.png and /dev/null differ
diff --git a/website/docs/modeling/load_data/images/streaming/h.png b/website/docs/modeling/load_data/images/streaming/h.png
deleted file mode 100644
index 907b67368e..0000000000
Binary files a/website/docs/modeling/load_data/images/streaming/h.png and /dev/null differ
diff --git a/website/docs/modeling/load_data/images/streaming/i.png b/website/docs/modeling/load_data/images/streaming/i.png
deleted file mode 100644
index 29a7805ea2..0000000000
Binary files a/website/docs/modeling/load_data/images/streaming/i.png and /dev/null differ
diff --git a/website/docs/modeling/load_data/images/streaming/j.png b/website/docs/modeling/load_data/images/streaming/j.png
deleted file mode 100644
index 4432bba537..0000000000
Binary files a/website/docs/modeling/load_data/images/streaming/j.png and /dev/null differ
diff --git a/website/docs/modeling/load_data/images/streaming/k.png b/website/docs/modeling/load_data/images/streaming/k.png
deleted file mode 100644
index 62b15380d8..0000000000
Binary files a/website/docs/modeling/load_data/images/streaming/k.png and /dev/null differ
diff --git a/website/docs/modeling/load_data/images/streaming/l.png b/website/docs/modeling/load_data/images/streaming/l.png
deleted file mode 100644
index 8187663277..0000000000
Binary files a/website/docs/modeling/load_data/images/streaming/l.png and /dev/null differ
diff --git a/website/docs/modeling/load_data/images/streaming/m.png b/website/docs/modeling/load_data/images/streaming/m.png
deleted file mode 100644
index bc47bd73ad..0000000000
Binary files a/website/docs/modeling/load_data/images/streaming/m.png and /dev/null differ
diff --git a/website/docs/modeling/load_data/images/streaming/n.png b/website/docs/modeling/load_data/images/streaming/n.png
deleted file mode 100644
index 1297831d51..0000000000
Binary files a/website/docs/modeling/load_data/images/streaming/n.png and /dev/null differ
diff --git a/website/docs/modeling/load_data/images/time_axis.png b/website/docs/modeling/load_data/images/time_axis.png
deleted file mode 100644
index 9a7b7d7d15..0000000000
Binary files a/website/docs/modeling/load_data/images/time_axis.png and /dev/null differ
diff --git a/website/docs/modeling/load_data/images/time_axis_customize_build.png b/website/docs/modeling/load_data/images/time_axis_customize_build.png
deleted file mode 100644
index 5b2a7bc375..0000000000
Binary files a/website/docs/modeling/load_data/images/time_axis_customize_build.png and /dev/null differ
diff --git a/website/docs/modeling/load_data/intro.md b/website/docs/modeling/load_data/intro.md
deleted file mode 100644
index 21c10c98ee..0000000000
--- a/website/docs/modeling/load_data/intro.md
+++ /dev/null
@@ -1,25 +0,0 @@
----
-title: Data Loading
-language: en
-sidebar_label: Data Loading
-pagination_label: Data Loading
-toc_min_heading_level: 2
-toc_max_heading_level: 6
-pagination_prev: null
-pagination_next: null
-keywords:
- - data loading
-draft: false
-last_update:
- date: 08/19/2022
----
-
-Load data calculates the source data based on the model and index definition. This chapter takes the sample data as an example to introduce two data load methods and processes:
-
-- full load
-- incremental load by date/time
-
-This chapter also introduces segment operations and settings, which are used to manage Segments.
-
-- Segment operation and settings
- - Segment Merge
diff --git a/website/docs/modeling/load_data/segment_operation_settings/intro.md b/website/docs/modeling/load_data/segment_operation_settings/intro.md
deleted file mode 100644
index f070040d3a..0000000000
--- a/website/docs/modeling/load_data/segment_operation_settings/intro.md
+++ /dev/null
@@ -1,94 +0,0 @@
----
-title: Segment Management
-language: en
-sidebar_label: Segment Management
-pagination_label: Segment Management
-toc_min_heading_level: 2
-toc_max_heading_level: 6
-pagination_prev: null
-pagination_next: null
-keywords:
- - segment management
-draft: false
-last_update:
- date: 08/19/2022
----
-
-Model (index group) consists of one or more segments. Each segment contains a range of data. Segment is created by building index or loading data with a selected data range on the partition columns.
-
-The main contents are as follows:
-
-
-### <span id="view">View Segment</span>
-
-User can access the Segment management interface by following these steps:
-
-1. Open **Data Asset -> Model** page, and click the model (index group) name.
-2. Select **Segment** tag.
-
-The model list page in the project is shown as below.
-
-
-
-Field description of the Segment list:
-
-- Start time: The start time of the data in the Segment. If it is loaded in full **"Full Load"** is displayed.
-
-- End time: The end time of the data in the Segment. If it is loaded in full **"Full Load"** is displayed.
-
-- Index: Indexes in this segment / Total indexes
-
-- Status: Segment status. You can find a detailed introduction [Segment Status](#status) section.
-
-- Last Updated Time: Segment last updated time.
-
-- Source Records: The source records of the data in the segment.
-
-- Storage: The storage size of the data in the segment.
-
- > Tip: When the tiered storage is turned on, the storage size of the data loaded into the tiered storage (ClickHouse) will be displayed.
-
-- Actions: The operation of the segment. Currently only **Show Detail** is supported.
-
-
-
-### <span id="status">Segment Status</span>
-
-You can view the segment status in the segment list. There are 6 types of segment statuses. See below:
-
-- **ONLINE**: Segment can serve the query by indexes loaded data or pushdown engine.
-- **WARNING**: The data in the segment has been loaded and can serve the query. However, the source data might be changed which might cause the data inconsistent. It's highly recommended to refresh all indexes within this segment.
-- **LOCKED**: Segments that are refreshing or merging will be locked.
-- **LOADING**: The data in the segment is loading.
-- **REFRESHING**: A new segment is automatically generated when you refresh the specified segment. This new segment is marked as *REFRESHING*. When the refresh is complete the old segment will be automatically deleted.
-- **MERGING**: A new segment is automatically generated when you merge the specified segments. This new segment is marked as *MERGING*. When the merge is complete the old segment will be automatically deleted.
-
-
-
-### <span id="expert_operation">Segment Operation</span>
-
-You have 6 types of segment operations on the **Data Asset -> model **page.
-
-- **+ Segment**: Add segments to define the model’s data range for serving queries. Queries within the range could be answered by indexes or pushdown engine. Queries out of the range would have no results. The button is located above the segment list.
-
-> **Note:** In **Setting -> Basic Settings -> Segment Settings**, enable **Creating Reserved Segments**, then the **+ Segment** operation button will appear.
-
-- **Show Detail**: You can click the icon on the right side of the segment list. When your mouse is hovering over the icon **Show Detail** is displayed. You can view details such as storage size, the data range and more.
-
-- **Refresh**: Refresh the data in the segment. This operation supports batch refresh. The **Refresh** button is located above the segment list.
-
- >**Note**: Only ONLINE and WARNING status segments can be refreshed.
-
-- **Merge**：Merge multiple segments as a new one. The **Merge** button is located above the segment list.
-
- > **Note**: Only ONLINE and WARNING status segments can be merged.
-
-- **Delete**: Delete the segment. This operation supports batch deletion. The **Delete** button is located above the segment list.
-
-- **Fix**: Fix the discontinuous segments. This button will be only displayed above the Segment list when the holes exists in Segment ranges.
-
-
-
-### <span id="setting"> Segment Settings</span>
-
-You can set some to manage segments automatically in the **Setting -> Segment Settings** page. Please refer to [Project Settings](../../../operations/project-managing/project_settings.md) for the specific requirements.
diff --git a/website/docs/modeling/manual_modeling.md b/website/docs/modeling/manual_modeling.md
deleted file mode 100644
index e3178b98d3..0000000000
--- a/website/docs/modeling/manual_modeling.md
+++ /dev/null
@@ -1,331 +0,0 @@
----
-title: Manual Modeling
-language: en
-sidebar_label: Manual Modeling
-pagination_label: Manual Modeling
-toc_min_heading_level: 2
-toc_max_heading_level: 6
-pagination_prev: null
-pagination_next: null
-keywords:
- - manual modeling
-draft: false
-last_update:
- date: 08/19/2022
----
-
-Kylin follows multidimensional modeling theory when building star or snowflake models based on your tables. Kylin also leverages pre-computation technique and will reuse the pre-computed results to answer queries, so there is no need to traverse all data when there is a query, and thus achieve sub-second query times on PB-level data.
-
-### **Operation steps**
-
-Kylin model consists of multiple tables and their join relations. In this article, we use this [SSB dataset](../quickstart/tutorial.md), a dataset based on real business applications, and hope to analyze products and supplier information from dimensions such as year, city, supplier name and brand. To achieve this goal, we will create a model in Kylin, and then set dimensions and measures for multidimensional analysis. See the table below for detailed steps.
-
-| **Step** | **Description** |
-| ------------------------------------------------------------ | ------------------------------------------------------------ |
-| Step 1: Create a model and add a fact table and dimension tables | In this step, we design the model, and then define the fact table and dimension tables to be analyzed, which will serve as data sources for later data analysis. |
-| Step 2: Create join relations among tables | Create join relations between the foreign keys of the fact table and the primary keys of the dimension tables to achieve join queries of the two tables. |
-| Step 3: Add dimensions and measures to the model | Set the dimensions and measures for data analysis. Kylin will run pre-computation based on the combination of the defined dimensions and measures, which will greatly accelerate data query. |
-| Step 4: Save the model and set the loading method | Save the model settings and specify the data loading method for the pre-computations step. If incremental load is selected, data within a specified time range will be loaded to improve loading efficiency. |
-
-### **Step 1: Create a model and add a fact table and dimension tables**
-
-1. Log in to Kylin as any of the following roles:
-
- - System admin
- - **Management** or **Admin** of the target project
-
-2. To create a model:
-
- 1. In the left navigation panel, click **Data Asset** > **Model**.
-
- 2. Click **+ Model**.
-
- 3. In the pop-up dialog box, enter a name and description for the model, and then click **Submit**.
-
- Model name can be any combination of numbers, letters, and underscores (_).
-
-3. You will be directed to the model editing page. On this page, add the fact table to the model.
-
- Fact table is used to store fact records, that is, to store data about a business process at the finest granularity, for example, product sales table. It often serves as the primary table of a model.
-
- 1. In the left-hand **Data Source** section, find the target fact table (**P_LINEORDER** in this example).
-
- > [!NOTE]
- >
- > If there is no table in the **Data Source** section, please [load data source](../datasource/intro.md) first.
-
- 2. Drag the target table to the right-hand canvas and select **Switch to Fact Table**.
-
- 
-
-4. Add dimension tables to the model.
-
- Dimension table, also called lookup table, is used to store repeated attributes of the fact table, such as date and geographic location. Dimension tables can help to reduce the fact table size and improve dimension management efficiency.
-
- 1. In the left-hand **Data Source** section, find the target dimension table.
-
- 2. Drag the target table to the right-hand canvas.
-
- To add multiple dimension tables, repeat this step for each table. As shown below, one fact table and 4 dimension tables are added.
-
- 
-
-### **Step 2: Create join relations among tables**
-
-1. On the model editing page, drag a column to create a join relation between the foreign key of the fact table and the primary key of the dimension table.
-
- 
-
-2. In the **Add Join Relationship** dialog box, follow the instructions below to set the join relation.
-
- 
-
- - **Join Relationship for Tables**: It includes 3 drop-down lists. The first and the third one specify the tables to be joined, and the second one defines the join relation. Kylin currently supports **LEFT** (left join) and **INNER** (inner join).
-
- - **Table Relationship:** Select the mapping between the foreign and primary keys: **One-to-One or Many-to-One**, or **One-to-Many or Many-to-Many**.
-
- - **Precompute Join Relationship**: Select whether to expand the joined tables into a flat table based on the mappings. This option is selected by default. For more information about this function and its applicable scenarios, see [Precompute the join relations](model_design/precompute_join_relations.md).
-
- - **Join Relationship for Columns**: It includes 3 drop-down lists. The first and the third one specify the columns to be joined, and the second one defines the join relation, which is equal-join (=) by default. Join relations should meet the following requirements:
- - Do not define more than one join relation for the same column; two tables could only be joined by the same condition for one time
- - Join relations for columns should include at least one equal-join condition (=)
- - Join relations ≥ and < must be used in pairs, and the column in between must be the same. Example: B ≥ A, C < A
-
-3. Click **OK**.
-
-To create join relations for multiple tables, repeat steps 1-3 for each table. In this example, we create 4 join relations for tables, which constitute a star model.
-
-
-
-Corresponding SQL statements:
-
-```SQL
-P_LINEORDER LEFT JOIN DATES ON P_LINEORDER.LO_ORDERDATE = DATES.D_DATEKEY
-P_LINEORDER LEFT JOIN CUSTOMER ON P_LINEORDER.LO_CUSTKEY = CUSTOMER.C_CUSTKEY
-P_LINEORDER LEFT JOIN SUPPLIER ON P_LINEORDER.LO_SUPPKEY = SUPPLIER.S_SUPPKEY
-P_LINEORDER LEFT JOIN PART ON P_LINEORDER.LO_PARTKEY = PART.P_PARTKEY
-```
-
-### **Step 3: Add dimensions and measures to the model**
-
-1. To add dimension tables to the model:
-
- 1. On the model editing page, drag dimension columns from dimension tables to the **Dimension** section.
-
- To add dimensions in batch, click **+** in the **Dimension** section.
-
- 
-
- 2. In the pop-up dialog box, set the dimension name.
-
- By default, it's the column name. It can be any combination of letters, numbers, spaces, and special characters `(_ -()%?)`.
-
- 3. Click **OK**.
-
- In our example, we added year (D_YEAR in DATE), the city customer is in (CITY in CUSTOMER), supplier name (S_NAME in SUPPLIER), and brand (P_BRAND in PART) as dimensions.
-
-2. Add measures to the model.
-
- 1. On the model editing page, drag dimension columns from dimension tables to the **Measure** section.
-
- To add dimensions in batch, click **+** in the **Measure** section.
-
- 
-
- 2. In the **Add Measure** dialog box, follow the instructions below to complete join relation settings.
-
- - **Name**: Column name by default. It can be any combination of letters, numbers, spaces, and special characters `(_ -()%?)`.
-
- - **Function**: **SUM (column)** by default. Kylin has a variety of built-in basic and advanced functions, such as Count Distinct, TopN, etc. For more information, see [Advanced measures](model_design/advance_guide/intro.md).
-
- - **Column**: The measure column. No adjustment is needed.
-
- - **Note** (Optional): Enter notes to facilitate future measure management.
-
- 3. Click **Submit**.
-
- In our example, we added revenue (LO_REVENUE in P_LINEORDER) and supply cost (LO_SUPPLYCOST in P_LINEORDER) as measures, and wanted to calculate the sum for each.
-
-3. (Optional) To achieve complex processing and computation based on the existing columns, you can add computed columns to the model. For more information, see [Computed columns](model_design/computed_column.md).
-
-### Step 4: Save the model and set the loading method
-
-1. In the bottom right corner of the model editing page, click **Save.**
-
-2. In the **Save** dialog box, follow the instructions below to complete model settings.
-
- 
-
- - **Please select a load method**:
- - **Full Load**: Load and pre-compute all data in the source table according to different combinations of dimensions and measures.
- - **Incremental Load**: Load and pre-compute data within the specified time range in the source table according to combinations of dimensions and measures. You also need to specify the following parameters if this option selected.
- - **Partition Table**: Fact table (default and cannot be changed)
- - **Time Partition Column**: Select a column of the time type in the partition table.
- - **Time Format**: Select the time format. Or you can click  and Kyligece will automatically fill in the time format.
- - **Advanced Setting**: Use the data filter to filter out null values or data meeting certain requirements. Use `AND` or `OR` to associate multiple filters, for example, `BUYER_ID <> 0001 AND COUNT_ITEM > 1000 OR TOTAL_PRICE = 1000`.
- - **Add Base Indexes**: Add the following base indexes. This option is enabled by default so base indexes will automatically update when model dimensions and measures change.
- - Base aggregate index: It includes all model dimensions and measures.
- - Base table index: It includes all columns of model dimensions and measures.
-
-3. Click **Submit**.
-
- After the model is saved, you can click **View Index** in the **Notice** dialog box to check the aggregate index and table index that Kylin automatically creates.
-
-### Next steps
-
-For the newly created base indexes, you need to [build them](load_data/build_index.md) so Kylin can run pre-computation based on these indexes to accelerate queries.
-
-> [!NOTE]
->
-> There are few scenarios where base indexes can be used to accelerate queries. To improve query efficiency, you need to add more indexes to the model. For more information, see [Aggregate index](model_design/aggregation_group.md) and [Table index](model_design/table_index.md).
-
-Model design refers to build the star model or snowflake model based on data table and multidimensional modeling theory. The main contents of model design are as follows:
-
-- Define Fact Table and Dimension Table
-- Define the Association Relationship between Tables
-- Define Dimension and Measurement
-
-### <span id="model">Model List</span>
-
-You can create and design models manually. Below are the main content of model list:
-
-1. Log in to Web UI, switch to a project.
-
-2. Navigate to **Data Asset -> Model** page, where models are shown in a list. The picture below is the index group list:
-
- 
-
- **Fields Explanation:**
-
- - **Model Name**: Model's name.
-
- - **Status**: There are four statuses.
- - *ONLINE* indicates this model is online and is able to answer queries.
- - *OFFLINE* indicates this model is offline and not available to answer queries. We recommend using offline when you need to edit the model.
- - *BROKEN* indicates this model is broken and not available. Mostly happens when the schemas of related source tables have changed, for instance, a related source table is deleted.
- - *WARNING* indicates this model is warning and can only server parts of queries. Mostly happens when the segments exist holes or indexes are waiting to build.
- - **Last Updated Time**: The lastest time to update model.
-
- - **More Actions**: The **More Actions** button will appear when you are hovering on model name area, please refer to [Model Operations](#operation) for details.
-
- - **Owner**: The user who created this model.
-
- - **Description**: Model description.
-
- - **Fact Table**: The fact table of this model.
-
- - **Types**: Model types, which include *Batch Model*
-
- - **Usage**: Hit count by SQL statements in the last 30 days. Update every 30 minutes.
-
- - **Rows**: The rows of loaded data in this model.
-
- - **Storage**: The storage size of loaded data in this model, which combines the storage size of all Segments data.
-
- > Tip: When the tiered storage is turned on, the total storage size of the data loaded into the tiered storage (ClickHouse) will be displayed.
-
- - **Expansion Rate**: The ratio of the storage size of the built data to the storage size of the corresponding source table data under the model. Expansion Rate = Storage Size / Source Table Size.
-
-
- > Notice: The expansion rate won't show if the storage size is less than 1GB.
-
- - **Index Amount**: The amount of indexes in this model.
-
-### <span id="operation">Model Operation</span>
-
-You are only allowed to operate on models. You can hover on the right most column **Actions** of the model list to get the action names. Specific actions are listed below:
-
-- **Edit**: Click on the pencil shape button, enter into the model editing page.
-
-- **Build Index**: Loads data for models. You can choose the data range in the pop-up window.
-
-- **Model Partition**: Set partition column for the model.
-
-- **Export Model**: Export single model metadata.
-
- > **Note**: Since the locked indexes will be deleted after the new indexes have been built, the exported model metadata will not include the locked index.
-
-- **Export TDS**: Export TDS file of the model .
-
-- **Rename**: Renames the model.
-
-- **Clone**: Clones an identical model. You can give a new name for this new model. The new model has the same fact table, dimension tables, join relationship, dimensions, measures, computed columns, date partition column, aggregate indexes, table indexes, etc. as the origin model. But the new model does not have data, you need to load data for this cloned model manually.
-
- > **Note**: Since the locked indexes will be deleted after the new indexes have been built, the cloned model will not include the locked index.
-
-- **Change Owner**：Change model owner. Only system administrators and project administrators have the authority to modify model owner.
-
-- **Delete**: Deletes the model, remove the loaded data at the same time.
-
-- **Purge**: Purges all loaded data in this model.
-
-- **Offline**: Makes a *Online / Warning* model offline. An offline model cannot answer any queries.
-
-- **Online**: Makes a *Offline* model online. An online model should be able to answer related queries.
-
-> **Note:** If the model is in *BROKEN* status, only the **delete** operation is allowed.
-
-
-### <span id="more">Model Details</span>
-
-Models contain Segments and indexes. You can click model name to unfold the detailed information, as shown below:
-
-
-
-- **Overview**: Check Overview details, please refer to [Model Overview](#overview) for more.
-- **Data Features**: Check data features.
-- **Segment**: Check Segment details, please refer to [Segment Operation and Settings](load_data/segment_operation_settings/intro.md) for more.
-- **Index**: Review the model indexes.
- - **Index Overview**: Check index overview.
- - **Aggregate Group**: Add or check defined aggregate indexes, please refer to [Aggregate Index](model_design/aggregation_group.md) for more details.
- - **Table Index**: Add or check defined table indexes, please refer to [Table Index](model_design/table_index.md) for more details.
-- **Developers**: Check information for developers.
- - **JSON**: Kylin describes the information of models (index groups) in `JSON` format, such as design, dimensions, measures, etc.
- - **SQL**: The SQL statement consists of related information about tables and columns in the model, such as the join conditions between the tables.
-
-### <span id="overview">Model Overview</span>
-
-After expanding the model information, you can see the model overview page, which will help you to quickly get the model information.
-
-
-
-On this page, you can view the ER diagram of the model.
-
-
-
-What's more, you can view the dimensions and measures information contained in the model.
-
-
-
-
-
-
-### FAQ
-
-- Question: Why did I get an error when saving the time partition column settings?
-
- Answer: This error occurs when the time format of the time partition column does not match the target format. Kylin supports the following time formats: `yyyyMMdd`, `yyyy-MM-dd`, `yyyy/MM/dd`, `yyyy-MM-dd HH:mm:ss`, `yyyy-MM-dd HH:mm:ss.SSS`, `yyyy-MM`, `yyyyMM` and `yyyy-MM-dd'T'HH:mm:ss.SSS'Z'`.
-
- Kylin also supports customized time formats if the following conditions are met:
-
- - Any combination of yyyy, MM, dd, HH, mm, ss, and SSS is used with the elements in ascending order.
- - Hyphens (-), forward slashes (/), columns (:), or spaces are used as separators.
- - Unformatted letters are enclosed in single quotation marks ('). For example, 'T' is recognized as T.
-
- > [!NOTE]
- >
- > When the time format is customized as `yyyyMMddHHmmss`, the corresponding column in the Hive table should be strings, or Kylin may fail to recognize column data.
-
-- Question: I've modified several tables from the same model. Why did I get an error when reloading these tables?
-
- Answer: Since Kylin currently only supports loading a single table at a time, please edit and reload tables one by one, rather than reload several modified tables at a time.
-
-- Question: I've modified several tables from the same model. Why did I get an error when reloading these tables?
-
- Answer: Since Kylin currently only supports loading a single table at a time, please edit and reload tables one by one, rather than reload several modified tables at a time.
-
-- Question: What are the rules for the model to go online?
-
- Answer: It will automatically switch to online when the building job completed. However, customer may need to continually build some historical data during a period when creating a new report or testing, and may not want this model to serve any query unless it has built all data. In this case, we also offer a model level configuration to ensure users can control the model status manually . After configuring the `kylin.model.offline` to `true` (default value is false) in model rewriting [...]
-
- 
diff --git a/website/docs/modeling/model_design/advance_guide/intro.md b/website/docs/modeling/model_design/advance_guide/intro.md
deleted file mode 100644
index f0db643d96..0000000000
--- a/website/docs/modeling/model_design/advance_guide/intro.md
+++ /dev/null
@@ -1,17 +0,0 @@
----
-title: Model Advanced Settings
-language: en
-sidebar_label: Model Advanced Settings
-pagination_label: Model Advanced Settings
-toc_min_heading_level: 2
-toc_max_heading_level: 6
-pagination_prev: null
-pagination_next: null
-keywords:
- - model advanced settings
-draft: false
-last_update:
- date: 09/13/2022
----
-
-This section describes the advanced settings applied to the model.
diff --git a/website/docs/modeling/model_design/aggregation_group.md b/website/docs/modeling/model_design/aggregation_group.md
deleted file mode 100644
index 72652281d8..0000000000
--- a/website/docs/modeling/model_design/aggregation_group.md
+++ /dev/null
@@ -1,414 +0,0 @@
----
-title: Aggregate Index
-language: en
-sidebar_label: Aggregate Index
-pagination_label: Aggregate Index
-toc_min_heading_level: 2
-toc_max_heading_level: 6
-pagination_prev: null
-pagination_next: null
-keywords:
- - aggregate index
-draft: false
-last_update:
- date: 09/13/2022
----
-
-
-Curse of dimensionality is an infamous problem for all OLAP engines based on precalculation. In Kylin, we designed the aggregation group mechanism to fix this issue.
-
-
-### <span id="background">Introduction</span>
-
-It is a known fact that Kylin speeds up query performance by pre-calculating aggregate index group (different combination of all dimensions, aka index). The problem is that indices grow exponentially with the dimension. For example, there are a total of 7 possible indices for an aggregate index group with 3 dimensions, however there are 15 possible cuboids for a aggregate index with 4 dimensions. Kylin is using scalable computation framework (Spark) and distributed storage (HDFS) to comp [...]
-
-
-
-
-
-
-To alleviate the pressure on index building, Kylin has released a series of advanced settings to help users filter on only those indices being used. These advanced settings include **Aggregate Group**, **Joint Dimension**, **Hierarchy Dimension**, and **Mandatory Dimension**.
-
-
-
-### <span id="edit">Edit Aggregate Index</span>
-
-In the **Data Asset -> Model** page, click the model name to get more information, you can click **Index**, then click **Aggregate Group** button under **+ Index** in the **Index Overview** tab to enter the aggregate index editing page. Or you can click **+** (Add Aggregate Group) button in the **Aggregate Group** tab to enter the page. Then you can edit the aggregate index in the pop-up window shown below, and define dimensions and measures in different aggregate groups according to yo [...]
-
-
-
-
-**Step 1: Dimension Setting**
-
-The initial interface is to edit *Aggregation Group 1*. First you need to set the dimension, select **include **, click **Edit** and select from the list all the dimensions included by *Aggregation Group 1*. The items in the list is the dimensions you added in model. Kylin will build different combinations of all the dimensions you choose, this is called "build aggregate index".
-
-Users can then set **Mandatory Dimension**, **Hierarchy Dimension**, and **Joint Dimension** in the *Aggregate Group 1*. Dimensions under these three settings have to be included in **Include** under this aggregate group first. You can add aggregate groups as needed. After editing all the aggregate groups, click the button beside **Index Amount** on the bottom left corner, estimated index number will be calculated and displayed beside the name of the aggregate group, the total estimated [...]
-
-
-
-We recommend selecting frequently paired grouping dimensions and filtering dimensions into the aggregate group according to the cardinality from high to low. For example, you often query for the supplier ID `SUPPKEY` and the product ID `PARTKEY`, by adding the dimension `SUPPKEY` and dimension `PARTKEY` into the aggregate group, you can view the cardinality of these two relevant columns in **Data Asset-Data Source**. If the cardinality of `SUPPKEY` is greater than the cardinality of `PAR [...]
-
-**Step 1: Measure Setting**
-
-In the measure settings, all measures defined in the model are selected by default. You can define the measures corresponding to the selected dimensions in the aggregate group according to the business scenario. The aggregate index contains only the measures selected in the aggregate group. Whether or not you manually add a measure to the model, there is at least one measure `COUNT_ALL` in the aggregate group, which means ` COUNT(*) `.
-
-
-
-Next, we will explain the mechanism of these advanced settings and provide use cases in the following sections.
-
-
-
-### <span id="agg">Aggregate Group</span>
-
-You can divide combination of dimensions and measures they focus on in several groups, and these groups are called **Aggregate Group.**
-
-As the aggregate index group shows at the begining of this chapter, if you only need dimension combination of `[Dimension AB - Measure M1]` and `[Dimension CD - Measure M2]`, then aggregate index group can be divided into two aggregate groups, group AB-M1 and group CD-M2. As shown below, the number of index can be reduced from 15 to 7.
-
-
-
-
-The aggregate groups that end users need might contain overlapping dimensions, for example, aggregate group ABC and aggregate group BCD both contain dimension B and C. If these two aggregate groups contain the same measures, these aggregate groups will derive the sample index, for example aggregate group ABC will derive index BC and so does aggregate group BCD. An index will not be generated multiple times, if it can be derived from more than one aggregate group, as shown below.
-
-
-
-
-
-According to your business scenario, you can decide whether to add an index that contains dimensions and measures defined in all aggregate groups. The index can answer queries across multiple aggregate groups, but this will impact query performance. In addition to this, there are some storage and building costs by adding this index. You can define it by clicking **+** (Add Setting Item) and setting the item `is-base-cuboid-always-valid` in **Setting -> Model Settings**.
-
-With aggregate groups, you can filter the granularity of index to get the dimensionality you want.
-
-
-
-#### Use Case of Aggregate Group
-
-Assume a transactional aggregate index, which contains these dimension: Customer ID `buyer_id`, Transaction Date `cal_dt`, Payment Type `pay_type`and Customer City `city`. Sometimes, analyst need to group dimension City, Cal_dt and Pay_Type to understand different payment type in different cities. There are other times analysts need to group dimension city, cal_dt andbuy_id together to understand customer behavior in different cities. As shown in the example above, we recommend building [...]
-
-
-
-
-
-
-Aggregate Group 1: `[cal_dt,city, pay_type]`
-
-Aggregate Group 2: `[cal_dt,city, buyer_id]`
-
-Regardless of other situations, those two aggregate groups can reduce 3 unnecessary index: [pay_type, buyer_id]、[city,pay_type, buyer_id] and [cal_dt, pay_type, buyer_id], so storage space and build time can be saved.
-
-
-Case 1:
-```sql
-Select cal_dt, city, pay_type, count(*) from table
-Group by cal_dt, city, pay_type
-```
-will hit on index [cal_dt, city, pay_type]
-
-
-Case 2:
-
-```sql
-Select cal_dt, city, buyer_id, count(*) from table
-Group by cal_dt, city, buyer_id
-```
-will hit on index [cal_dt, city, buyer_id]
-
-Case 3: If one unusual query occur
-
-```sql
-Select pay_type, buyer_id,count(*) from table
-Group by pay_type, buyer_id
-```
-No index can be hit, so Kylin will calculate the result based on existing index on-demand.
-
-
-
-### <span id="mandatory">Mandatory Dimension</span>
-
-Sometimes you might be interested in analysis with one or more specific dimensions, any queries will include one specific dimension. In this case, this dimension can be set as **Mandatory Dimension**. Once set only the index with this dimension will be calculated. In the example at the begining of this chapter, if dimension A is set as mandatory, then index will be calculated as shown below. The number of index will be reduced from 15 to 8.
-
-
-
-
-
-
-
-#### Use Case of Mandatory Dimension
-Assume a transactional aggregate index that includes transaction date, transaction location, product, payment type. Transaction date is a frequently used group by dimension. If transaction date is set as a mandatory dimension, the combination of dimensions will be calculated as shown below:
-
-
-
-
-
-### <span id="hierarchy">Hierarchy Dimension</span>
-
-End users will usually use dimensions with hierarchical relationship, for example, country, province, and city. In this case, a hierarchical relationship can be set as **Hierachy Dimension**. From top to bottom, country, province and city are one-to-many relationship. These three dimensions can be grouped into three different combinations：
-
-1. group by country
-
-2. group by country, province（equivalent to group by province）
-
-3. group by country, province, city（equivalent to group by country, city or group by city）
-
- In the aggregate index group shown below, assume dimension A = Country, dimension B = Province and dimension C = City, then dimension ABC can be set as a hierarchy dimension. And index [A, C, D] = index [A, B, C, D]，index [B, D] = index [A, B, D], thus, index [A, C, D] and index [B, D] can be pruned.
-
- 
-
- As the diagram below illustrates, based on the method above, Kylin can prune redundant index, hence reducing index from 15 to 7.
-
- 
-
-
-#### Use Case of Hierarchy Dimension
-Assume a transactional aggregate index that includes dimensions transaction city `city`, transaction province `province`, transaction country `country` and payment type `pay_type`. Analysts will group transaction country, transaction province, transaction city, and payment type together to understand customer payment type preference in different geographical locations. In the example above, we recommend creating hierarchy dimensions in existing aggregate group (Country / Province / City) [...]
-
-
-
-Aggregate Group: `[country, province, city，pay_type]`
-
-Hierarchy Dimension: `[country, province, city]`
-
-
-Case 1:
-
-Analysts want to understand city level customer payment type preferences
-
-```sql
-SELECT city, pay_type, count() FROM table GROUP BY city, pay_type
-```
-
-can be retrieved from index [country, province, city, pay_type].
-
-
-
-Case 2:
-
-Analysts want to understand province level customer payment type preferences
-
-```sql
-SELECT province, pay_type, count() FROM table GROUP BY province, pay_type
-```
-
-can be retrieved from index [country, province, pay_type].
-
-
-
-Case 3:
-
-Analysts want to understand customer's payment type preferences from country level
-
-```sql
-SELECT country, pay_type, count() FROM table GROUP BY country, pay_type
-```
-
-can be retrieved from index [country, pay_type].
-
-
-
-Case 4:
-
-Analysts want to reach a different granularity level using the geographical dimension:
-
-```sql
-SELECT country, city, count(*) FROM table GROUP BY country, city
-```
-
-
-will retrieve data from index [country, province, city].
-
-
-
-### <span id="joint">Joint Dimension</span>
-
-Sometimes you don’t need details of all possible combination dimensions. For example, you might query dimension A, B, C together in most cases but not dimension A, C or dimension C alone. To enhance performance in this case, **Join Dimension** can be used. If A, B, and C are defined as Joint Dimension, Kylin will only build index ABC but not index AB, BC and A. Finally, indices will be built as below. The number of indices can then be reduced from 15 to 3.
-
-
-
-
-#### Use Case of Joint Dimension
-Assume a transactional aggregate index that includes dimension transaction date `cal_dt`, transaction city `city`, customer gender `sex_id`, payment type `pay_type`. Analysts usually need to group transaction date, transaction city, and customer gender to understand consumption preference for different genders in different cities, in this case, `cal_dt, city, sex_id `will be grouped together. In the case above, we recommend assigning them to joint dimensions based on existing aggregate g [...]
-
-
-
-Aggregate Group: `[cal_dt,city, sex_id，pay_type]`
-
-Joint Dimension: `[cal_dt, city, sex_id]`
-
-
-
-Case 1:
-
-```sql
-SELECT cal_dt,city, sex_id, count(*) FROM table GROUP BY cal_dt, city, sex_id
-```
-can retrieve data from index [cal_dt, city, sex_id].
-
-Case 2:
-
-```sql
-SELECT cal_dt, city, count(*) FROM table GROUP BY cal_dt, city
-```
-then no index can be hit, Kylin will leave calculate result based on existing index.
-
-**Dimensional cardinality product **
-
-Dimensional cardinality product: refers to the product of each dimension field cardinality in the joint dimension. The data of dimension cardinality comes from the sampling results of the source data table. The dimension cardinality product is used to represent the maximum number of permutations and combinations of dimensions in the joint index (i.e. the maximum number of pieces of this index).
-
-The dimension cardinality product does not participate in the index creation process, but is only used to assist in the creation of the union dimension of the aggregate group. In general, in order to ensure the query performance of the dimensions in the union dimension, the dimension cardinality product of a union dimension is not recommended to exceed 100,000, and in special cases (the dimensions in the union dimension must be queried together) can not pay attention to the value of the [...]
-
-
-### <span id="mdc">Max Dimension Combination</span>
-
-The use of aggregate groups helps to avoid index number explosion. However, in order to achieve index optimization, modelers need to have a certain understanding of the data model, which is hard for the junior modelers.
-
-This chapter will introduce another simple index pruning tool named *Max Dimension Combination (MDC)*, which represents the maximum number of dimensions in every index. This tool limits the dimension number in a single index, which means indexes containing too many dimensions will not be built in index building process. This tool fits well in the situation where most queries only touch no more than N dimensions, where N is the MDC threshold that is configurable.
-
-> **Note**: MDC is only available from version 5.0.
-
-#### Dimensions Count in Query
-
-Next we introduce the method of counting the number of dimensions in a query. The number of dimensions in a query means the number of dimensions in the corresponding index of the query. For ordinary dimensions, one dimension is counted as 1. We treat a group of joint dimension or hierarchy dimension as one dimension when counting dimensions in a query, and ignore mandatory dimensions. For example,
-
-```sql
-select count(*) from table group by column_mandatory, column_joint1, column_joint2, column_hierarchy1, column_hierarchy2, column_normal
-```
-
-There is one mandatory dimension, two dimensions belonging to one joint dimension, two dimensions belonging to one hierarchy dimension and one normal dimension. So we treat them as 3 dimensions in index pruning.
-
-#### Schematic Diagram of Pruning
-
-
-
-This is an index sprouting graph which has 7 dimensions and some details are hidden in order to help understanding.
-
-When MDC = 4, the indexes which have over 4 dimensions will be pruned, such as *ABCDEF*, *ABCDEG*, *ABCDE*, and *ABCDF*.
-
-When MDC = 3, the indexes which have over 3 dimensions will be pruned, such as *ABCDEF*, *ABCDEG*, *ABCD*, *ABCE*.
-
-Considering the performance in index building, if you choose to generate an index in the model settings that contains dimensions and measures defined in all aggregate groups, this index will not be pruned. At the same time, according to the calculation method of query dimensions mentioned before, we treat a group of joint dimension or hierarchy dimension as one dimension, and ignore mandatory dimensions. Therefore, when using the pruning tool, it is necessary to consider the actual numbe [...]
-
-#### Set Max Dimension Combination (MDC)
-
-We'll introduce how to set Max Dimension Combination (MDC) in this section. Click **+** (Add Aggregation Group), you can set **Aggregate index-level MDC** for all aggregate groups and **Aggregate group-level MDC** for a single aggregate group on the Edit Aggregation Index page, as shown in the figure below. Enter a positive integer in the input box and click OK to save the MDC setting.
-
-> **Note**: The setting of MDC will not take effect until you submit the Edit Aggregate Index page.
-
-
-
-Aggregate index-level MDC affects all aggregate groups. Aggregate group-level MDC takes effect only for a single aggregate group. The priority of the aggregate group-level level MDC is higher than that of aggregate index-level MDC. The specific setting rules are as follows:
-
-1. If there is no MDC setting for a single aggregation group, the MDC of this aggregate group will be overridden by the aggregate index-level MDC. It can be seen that after the aggregate index-level MDC is set to 2, the number of indexes of the aggregate group decreases from 32 to 19, and the total number of indexes becomes 20. Except for an index that contains dimensions and measures defined in all aggregate groups, indexes that contain more than 2 dimensions are pruned.
-
- 
-
-2. If a single aggregate group has set its aggregate group-level MDC, the number of index dimensions for this aggregate group is limited only by its own MDC.
-
- 
-
-3. If the Aggregate index-level MDC is left blank, it means that you have turned off the setting of the MDC for all aggregate groups. There is no limit to the number of dimensions in the index generated by an aggregation group that has not individually set the aggregate group-level MDC at this time. Aggregate groups that have set the aggregate group-level MDC are not affected.
-
-#### Benefit and Trade-off
-
-On one hand, MDC dimension pruning tool reduces the index number and storage size significantly. On the other hand, some complex queries that cover more dimensions may hit large indexes, hence online calculation cannot be avoided, which may make query response slower. Like other index optimization tools, it's a kind of trade-off. If most queries touch fewer dimensions in your case, MDC deserves a shot.
-
-At the same time, according to laboratory test data, it is found that when the number of dimensions in the aggregate group is large, it may take several minutes to check index amount. During this period, there may be web UI lags, please be patient. The following are laboratory test data results for reference only. Please judge according to the actual scene:
-
-- 1 aggregate group with 1000 dimensions. When the aggregate index-level MDC is set to 1, the average time for checking index amount is 3.1 minutes.
-- 1 aggregate group with 1500 dimensions. When the aggregate index-level MDC is set to 1, the average time for checking index amount is 13.9 minutes.
-- 3 aggregate groups, each containing 500 dimensions. When the aggregate index-level MDC is set to 1, the average time for checking index amount is 3 minutes.
-
-### <span id="index">View Aggregate Index</span>
-
-Click **Data Asset -> Model** in the left navigation bar to enter the **Model List** page, then click the model name and you can view the details of aggregate index in **Index-Index Overview**.
-
-
-
-You can enter the index ID or index content in the **Search** box to filter the index. Index content includes the dimensions, measures, and columns contained in the index, the index content is fuzzy filtering, and the index ID is precisely filtered. You can click the button in the **Content** column to view the index details. What's more. you can edit and delete Custom(Table Index), and for other kinds of indexes, you can only delete.
-On the top of the list, you can see data range. Once you have already build index, it will show the time range of the data loaded in the index.If it is **full load**, the data range will show **full load**.
-
-If you need to delete some indexes, first you can select the indexes to be deleted, then click the **Delete** button above the list to delete.
-
-If you need to add base indexes when there is no base index or when a base index is missing. Click **+ Index** to add the base indexes in the drop-down box.
-
-**Fields Explanation:**
-
-- Index ID: Index ID.
-
-- Storage: The storage size of the precomputed data in the index.
-
-- Usage: The frequency of queries having hit this index.
-
-- Source: **Recommended** indicates that the index is derived from the system recommendation, and **Custom** indicates that the index is defined by user.
-
-- Status: The status of the index, divided into four types:
-
- - NO BUILD: Index not yet been built. You can click the **Build Index** button to build all NO BUILD indices;
-
- - BUILDING: Index being built. You can see the corresponding building job being executed in the job monitor page;
-
- - ONLINE: Index has been built and can serve queries;
-
- - LOCKED: Due to the modification of the aggregate group, the index to be deleted may be in LOCKED status. It can still serve queries.
-
- > Tip: When you modify an aggregate group and save it, some aggregate indices may be refreshed. For example, adding a measure to an aggregate group will refresh all the aggregate indices corresponding to the aggregate group. In Kylin, the corresponding behavior is to delete the original indices and add new indices. In order to ensure that your service is available, especially the query service is available, the original indices will be placed in LOCKED status before the new indices a [...]
- >
- > If modifying the aggregate groups will only cause some indices to be deleted, there will be no LOCKED indices. An index cannot be restored after it has been deleted.
-
-- Content: Index content.
-
-- Action: The actions that can be performed on the current index, such as **Build Index**.
-
-The details page shows the combination of dimensions and all measures in the index. For aggregate indexes, the order is the dimension first, the measure is after, the dimension appears as table. column, and for the table index, you will see all the columns in this index, the order, and the shardby columns.
-
-
-
-**Fields Explanation:**
-
-- Last Updated Time: The last time to update the index.
-- Content: Dimension name or measure name or columns.
-- Type: Dimension or Measure.
-- Cardinality: Cardinality of the column, which can be fetched by table sampling.
-- Shard by: ShardBy column of the index.
-
-### <span id="advanced">Advanced Settings</span>
-
-In the aggregate index, you can set the ShardBy column, and the data will be stored according to the column fragmentation to improve query efficiency. You can set the filter dimension or grouping dimension frequently used in the query as the ShardBy column according to the cardinality from high to low. In current version, you can only set one ShardBy column.
-
-> Note: The ShardBy column is applied to all custom aggregate indexes.
-
-In the navigation bar **Data Asset -> Model ** page, click the icon to the left of the specified model to expand the model for more information. You can see the **Advanced Setting** button in the **Index-> Aggregate Group**.
-
-In the **Advanced Setting**, you can select the dimensions that need to be set as the ShardBy column.
-
-Updating the ShardBy column will invalidate all aggregate indices that contain ShardBy columns and rebuild these indices to take effect. Checking **Build index now** when saving will trigger a new index build job. If not checked, you can click **Build** button to create building.
-
-
-
-#### Optimize Join with ShardBy Column
-Sometimes the subqueries of a join can match two indexes, Kylin will get data from the two indexes and do the join operation. When the cardinality of the subquery is high, the join operation may be time consuming. And user can optimize the join operation in such case by making the join key the ShardBy column.
-
-To enable this feature (it is enabled by default), user need to edit `kylin.properties` file and add the following line
-```
-kylin.storage.columnar.expose-sharding-trait=true
-```
-When this feature is enabled, Kylin will expose the sharding information to Spark. Join two large datasets in spark usually requires a data shuffle infront to make the data hash partitioned. As the ShardBy column already sharding the data, the shuffle stage can be skipped during the Join.
-
-##### Limitations
-1. As there can only be one ShardBy column for one model, the optimization applys for join on the single ShardBy column only.
-
-##### Example
-```
-select org_id, cust_id, sum1, sum2
-from (
- select org_id, cust_id, sum(....) sum1
- from fact
- where dt = ...
- group by org_id, cust_id
-) T1
-inner join (
- select cust_id, sum(...) sum2
- from fact
- where dt = ...
- group by cust_id
-) T2 on T1.cust_id = T2.cust_id
-```
-The above SQL can be queried by joining two aggregate indexes. To optimize the join operation, user can set the join key - `cust_id` as the ShardBy column. In that case the shuffle stage can be skipped for the join operation.
-
diff --git a/website/docs/modeling/model_design/computed_column.md b/website/docs/modeling/model_design/computed_column.md
deleted file mode 100644
index 8e0dba4968..0000000000
--- a/website/docs/modeling/model_design/computed_column.md
+++ /dev/null
@@ -1,204 +0,0 @@
----
-title: Computed Column
-language: en
-sidebar_label: Computed Column
-pagination_label: Computed Column
-toc_min_heading_level: 2
-toc_max_heading_level: 6
-pagination_prev: null
-pagination_next: null
-keywords:
- - computed column
-draft: false
-last_update:
- date: 09/13/2022
----
-
-
-**Computed Column** allows you to predefine actions such as data transformation and calculation in models. It enhances the data semantic abstraction according to various business scenarios. By replacing runtime calculation with offline index construction, the pre-calculation capability of Kylin is fully utilized. As a result, query performance willl improve significantly. It supports Hive UDF in computed columns, so existing business codes can be reused.
-
-This section will describe the supported computed columns in Kylin, which includes the basic concepts, usage of computed columns and supported functions in computed columns.
-
-- [Basic Concepts](#concept)
-- [Create Computed Columns](#create)
-- [Edit Computed Columns](#edit)
-- [Use Computed Columns in Indices](#index)
-- [Query on Computed Column](#query)
-- [Nested Computed Columns](#nested)
-- [Advanced Functions](#function)
-
-### <span id="concept">Basic Concepts</span>
-
-- **Naming convention:**
-
- Only letters, digits and underscores are supported. Pure digits or starting with underscores are NOT allowed.
-
-- **Expression:**
-
- Calculating logic. Expression of the computed column can contain columns across a fact table and a lookup table in current model.
-
-- **Re-use and reference:**
-
- - Computed columns are separated by model and implicitly subjected to a fact table, thus, one expression cannot generate multiple computed columns on different models with different names.
- - Strictly, computed columns cannot be reused across models in the same project. If you want to reuse a computed column in another model, you need to create a computed column with the same name and expression in the model.
- - Nested computed column is also supported too, i.e. you can use available computed column to produce a new one.
-
-- **Special notes:**
-
- One expression cannot generate multiple computed columns. Say we have two models `M1` and `M2` in the same project:
-
- - If both `M1` and `M2` share the same fact table `T1`, and the computed column `CC1 = T1.C1 + T1.C2` is defined on `M1`, you can create a computed column `CC1 = T1.C1 + T1.C2` rather than `CC2 = T1.C1 + T1.C2` on `M2`;
- - If both `M1` and `M2` share the same fact table `T1`, and the computed column `CC1 = T1.C1 + T1.C2` is defined on `M1`, you can create a computed column `CC2 = T1.C1 * T1.C2` rather than `CC2 = T1.C1 + T1.C2` on `M2`;
-
-- **Others:**
- - Computed columns can only be defined on the fact table by default. Only column expression, not column name, can be edited again after being created.
- - Computed column can use either columns of fact table or dimension table. However, if columns in different table share the same name, please make sure that used columns are in the form of `table_alias.column_name`.
- - Unexpected errors may occur when the computed column share the same name with other columns.
- - Aggregate functions such as `SUM`, `MIN`, `MAX`, etc. are not allowed when defining the computed column expression.
- - Expression only containing constants is not recommended to create computed column, such as `POWER(CAST(2 AS DOUBLE), 2)`.
- - If a function expression contains a column using the keywords as column name, it cannot be recommended as a computed column. For more details about keywords, please refer the **Keywords** section in [SQL Specification](../../query/specification/sql_spec.md)
- - Computed columns only support `Hive` data source in current version.
- - Computed columns cannot be used as time partition columns and join keys.
- - If a function expression contains a table name or column name that is not started with letters or includes special characters, need to double quote this name. For example: `"100_KYLIN_SALES"."100_PRICE" * "100_KYLIN_SALES"."200_ITEM_COUNT"`
-
-### <span id="create">Create Computed Column</span>
-
-To help you master how to create a computed column, we demonstrate an example scenario below. Say you have a fact table named `P_LINEORDER` with following columns:
-
-- `LO_EXTENDEDPRICE`: transaction price
-- `LO_QUANTITY` transaction quantity
-- `LO_ORDERDATE` transaction date
-
-We want to define two computed columns on this fact table, `T_PRICE_PER_ITEM` to calculate the total transaction price for each item, and `YEAR_OF_ORDER` to indicate the year of the order. Concrete expressions are as follows:
-
-- `T_PRICE_PER_ITEM = P_LINEORDER.LO_EXTENDEDPRICE * P_LINEORDER.LO_QUANTITY`
-- `YEAR_OF_ORDER = YEAR(P_LINEORDER.LO_ORDERDATE)`
-
-First, click the button **CC** marked in the picture. Then a window of **Computed Column** will pop up.
-
-
-
-Secondly, click the button **+** in this window, and a dialog box of **Add Computed Column** will pop up. Please fill in the following information:
-
-- `Column name`: Defines the name of the computed column.
-- `Expression`: Calculates the calculation logic for the column.
-
-
-
-Third, click the button **Submit**, the system will verify whether the name or the expression of the computed column is legal. If anything is wrong, the system will give you a tip, please correct it and resubmit. Once the computed column is created, you will see it on the fact table. As shown below, `T_PRICE_PER_ITEM` appears in the fact table `P_LINEORDER`.
-
-
-
-Finally, after creating the computed column, click the button **+** in the window of **Dimension** to add a new dimension which is based on the computed column `YEAR_OF_ORDER`, as shown below:
-
-
-
-Also, click the button **+** in the window of **Measure** to add a new measure `TOTAL_PRICE` which is based on computed column `T_PRICE_PER_ITEM`, as shown below:
-
-
-
-
-### <span id="edit">Edit Computed Columns</span>
-
-In some cases, we need to change the expression to adapt some business scenario changes. At this point, we can modify the expressions of the computable columns directly by editing the model.
-
-
-
-However, there are few limitations and attentions. Please read the following limitations carefully before using it:
-
-- **NOT** support modifying the name of computed column
-- **NOT** support modifying the nested computed column. If the computed column has been used as a nested computed column. It will be failed to modify the expression and the following message will be shown: `model [model_name] nested computed column [column_name] still contains computed column [column_name]`.
-- The changes might rebuild **related indexes** under model, which will pop up a message for users to confirm.
-- The changes might cause measures to be invalid, then delete the measures, related aggregation groups and layouts, which will also pop up a message for users to confirm.
-
-### <span id="index">Use Computed Columns in Indices</span>
-
-Now we have defined two computed columns in our mode and produced a new dimension and a new measure. If we want to take advantage of precalculation of index, we need to use it when creating indexes.
-
-You can use computed columns in an aggregate index or in a table index. Let's take an example.
-
-Firstly, click the model name to get more information in the **Data Asset->Model** page. Then we need to click **Index** to enter the **Index Overview** page as shown below. Click **+ Index** to add index.
-
-
-
-After the submission is successful, we have completed the basic usage about computed columns including:
-
-* Creates a computed column.
-* Creates dimensions and measures based on computed columns.
-* Defines computed columns in the index.
-
-### <span id="query">Query on Computed Column</span>
-
-A computed column is logically appended to the table's column list after creation. You can query the computed column as if it was a normal column as long as it is precalculated in an index.
-
-In order to improve query performance with computed columns, you need to define computed columns when creating an index.
-
-1. **Query Pushdown**
-
- If a computed column neither used as a dimension nor defined in indices, the query performance will not be improved. However, if **QUERY PUSHDOWN** is enabled, users can also use this computed column. Specifically, Kylin will analyze and translate this query into a queryable SQL to calculation engine.
-
- Say if there defines a computed column named `T_PRICE_PER_ITEM`, corresponding expression is `LO_EXTENDEDPRICE * LO_QUANTITY`, then if you query the SQL statement below:
-
- ```sql
- select sum(T_PRICE_PER_ITEM) from SSB.P_LINEORDER
- ```
-
- It will be translated to a new queryable SQL as below:
-
- ```sql
- select sum(LO_EXTENDEDPRICE * LO_QUANTITY) from SSB.P_LINEORDER
- ```
-
- Then, it will be pushed down to the calculation engine.
-
- > **Note** : if you wish to query the computed column, the complete join relations defined in the model that defines the computed column must be supplied in the SQL.
-
-2. **Explicit Query**
-
- If the name of computed column appears as a field or a parameter of functions, in a SQL statement. We call it **Explicit Query** on computed columns. For example:
-
- ```sql
- select sum(T_PRICE_PER_ITEM) from SSB.P_LINEORDER
- ```
-
-3. **Implicit Query:**
-
- If the expression of computed column appears as a field or a parameter of functions, in a SQL statement. We call it **Implicit Query** on computed columns. For example:
-
- ```sql
- select sum(LO_EXTENDEDPRICE * LO_QUANTITY) from SSB.P_LINEORDER
- ```
-
- In Kylin 4.x, expression of `LO_EXTENDEDPRICE * LO_QUANTITY` will be converted to `T_PRICE_PER_ITEM`. Then the original query will be translated to the query as shown below:
-
- ```sql
- select sum(T_PRICE_PER_ITEM) from SSB.P_LINEORDER
- ```
-
- If measure `sum(T_PRICE_PER_ITEM)` has been precalculated in an Aggregate Index, the query performance will be greatly improved.
-
-
-
-
-### <span id="nested">Nested Computed Columns</span>
-
-Expression of computed column can be nested by other computed columns. You can define a new computed column based on an existing computed column; this is called **Nested Computed Column**. The expression specification for nested computed columns is the same as normal computed columns.
-
-Here, we will introduce how to create a nested computed column. For example, we want to create column `D_PRICE_PER_ITEM = 2 * T_PRICE_PER_ITEM`. `D_PRICE_PER_ITEM` is the name of this nested computed column and `T_PRICE_PER_ITEM` is a predefined computed column.
-
-First, define a computed column named `T_PRICE_PER_ITEM`, click **Submit**
-
-
-Second, define a computed column named `D_PRICE_PER_ITEM` and the expression is `2 * T_PRICE_PER_ITEM`.
-
-
-
-After clicking **Submit** button, it will automatically validate the computed column expression. Then you will see the computed column information shown in the figure above.
-
-When creating nested computed columns, please make sure that the input computed column actually exists and the name is correct. Otherwise you will receive a prompt message that `Computed Column ${Computed Column Name} use nonexistent column(s):${Column Name}`. You can re-submit after correction.
-
-
-
-### <span id="function">Advanced Functions</span>
-
-Computed column is pushed down to data source and Spark SQL is the query push-down engine in Kylin 4.x. Thus, the syntax of computed column expression should follow the grammar of Spark SQL.
diff --git a/website/docs/modeling/model_design/images/agg/AGG-1.png b/website/docs/modeling/model_design/images/agg/AGG-1.png
deleted file mode 100755
index d24bce66fe..0000000000
Binary files a/website/docs/modeling/model_design/images/agg/AGG-1.png and /dev/null differ
diff --git a/website/docs/modeling/model_design/images/agg/advanced-EN-01.png b/website/docs/modeling/model_design/images/agg/advanced-EN-01.png
deleted file mode 100644
index 9664e82894..0000000000
Binary files a/website/docs/modeling/model_design/images/agg/advanced-EN-01.png and /dev/null differ
diff --git a/website/docs/modeling/model_design/images/agg/advanced-EN-02.png b/website/docs/modeling/model_design/images/agg/advanced-EN-02.png
deleted file mode 100644
index 71a9cdf6e1..0000000000
Binary files a/website/docs/modeling/model_design/images/agg/advanced-EN-02.png and /dev/null differ
diff --git a/website/docs/modeling/model_design/images/agg/advanced.png b/website/docs/modeling/model_design/images/agg/advanced.png
deleted file mode 100644
index 0110e22219..0000000000
Binary files a/website/docs/modeling/model_design/images/agg/advanced.png and /dev/null differ
diff --git a/website/docs/modeling/model_design/images/agg/agg-group-1.png b/website/docs/modeling/model_design/images/agg/agg-group-1.png
deleted file mode 100644
index 204e80c027..0000000000
Binary files a/website/docs/modeling/model_design/images/agg/agg-group-1.png and /dev/null differ
diff --git a/website/docs/modeling/model_design/images/agg/agg-group-2.png b/website/docs/modeling/model_design/images/agg/agg-group-2.png
deleted file mode 100644
index fbf8dfc627..0000000000
Binary files a/website/docs/modeling/model_design/images/agg/agg-group-2.png and /dev/null differ
diff --git a/website/docs/modeling/model_design/images/agg/agg-group-3.png b/website/docs/modeling/model_design/images/agg/agg-group-3.png
deleted file mode 100644
index 84b0886ddd..0000000000
Binary files a/website/docs/modeling/model_design/images/agg/agg-group-3.png and /dev/null differ
diff --git a/website/docs/modeling/model_design/images/agg/agg-group-4.png b/website/docs/modeling/model_design/images/agg/agg-group-4.png
deleted file mode 100644
index 972bdbd443..0000000000
Binary files a/website/docs/modeling/model_design/images/agg/agg-group-4.png and /dev/null differ
diff --git a/website/docs/modeling/model_design/images/agg/agg_1.png b/website/docs/modeling/model_design/images/agg/agg_1.png
deleted file mode 100644
index 5fb368a8b1..0000000000
Binary files a/website/docs/modeling/model_design/images/agg/agg_1.png and /dev/null differ
diff --git a/website/docs/modeling/model_design/images/agg/agg_2.png b/website/docs/modeling/model_design/images/agg/agg_2.png
deleted file mode 100644
index 2f0023259b..0000000000
Binary files a/website/docs/modeling/model_design/images/agg/agg_2.png and /dev/null differ
diff --git a/website/docs/modeling/model_design/images/agg/agg_index_2.png b/website/docs/modeling/model_design/images/agg/agg_index_2.png
deleted file mode 100644
index ae32a34a2c..0000000000
Binary files a/website/docs/modeling/model_design/images/agg/agg_index_2.png and /dev/null differ
diff --git a/website/docs/modeling/model_design/images/agg/agg_measure.png b/website/docs/modeling/model_design/images/agg/agg_measure.png
deleted file mode 100644
index d5600ba190..0000000000
Binary files a/website/docs/modeling/model_design/images/agg/agg_measure.png and /dev/null differ
diff --git a/website/docs/modeling/model_design/images/auto_modeling/convert_no.en.png b/website/docs/modeling/model_design/images/auto_modeling/convert_no.en.png
deleted file mode 100644
index 7981e6a728..0000000000
Binary files a/website/docs/modeling/model_design/images/auto_modeling/convert_no.en.png and /dev/null differ
diff --git a/website/docs/modeling/model_design/images/auto_modeling/convert_or_not.en.png b/website/docs/modeling/model_design/images/auto_modeling/convert_or_not.en.png
deleted file mode 100644
index 2ae045a66d..0000000000
Binary files a/website/docs/modeling/model_design/images/auto_modeling/convert_or_not.en.png and /dev/null differ
diff --git a/website/docs/modeling/model_design/images/auto_modeling/convert_yes_model.en.png b/website/docs/modeling/model_design/images/auto_modeling/convert_yes_model.en.png
deleted file mode 100644
index 4584b150f9..0000000000
Binary files a/website/docs/modeling/model_design/images/auto_modeling/convert_yes_model.en.png and /dev/null differ
diff --git a/website/docs/modeling/model_design/images/auto_modeling/convert_yes_recommend_en.png b/website/docs/modeling/model_design/images/auto_modeling/convert_yes_recommend_en.png
deleted file mode 100644
index 6ed4c8d4ae..0000000000
Binary files a/website/docs/modeling/model_design/images/auto_modeling/convert_yes_recommend_en.png and /dev/null differ
diff --git a/website/docs/modeling/model_design/images/auto_modeling/import_sql.en.png b/website/docs/modeling/model_design/images/auto_modeling/import_sql.en.png
deleted file mode 100644
index 9d4bc8e29b..0000000000
Binary files a/website/docs/modeling/model_design/images/auto_modeling/import_sql.en.png and /dev/null differ
diff --git a/website/docs/modeling/model_design/images/computed_column/cc_1.png b/website/docs/modeling/model_design/images/computed_column/cc_1.png
deleted file mode 100644
index f1f261c195..0000000000
Binary files a/website/docs/modeling/model_design/images/computed_column/cc_1.png and /dev/null differ
diff --git a/website/docs/modeling/model_design/images/computed_column/cc_11.png b/website/docs/modeling/model_design/images/computed_column/cc_11.png
deleted file mode 100644
index 973aaee590..0000000000
Binary files a/website/docs/modeling/model_design/images/computed_column/cc_11.png and /dev/null differ
diff --git a/website/docs/modeling/model_design/images/computed_column/cc_2.png b/website/docs/modeling/model_design/images/computed_column/cc_2.png
deleted file mode 100644
index 93c7919840..0000000000
Binary files a/website/docs/modeling/model_design/images/computed_column/cc_2.png and /dev/null differ
diff --git a/website/docs/modeling/model_design/images/computed_column/cc_3.png b/website/docs/modeling/model_design/images/computed_column/cc_3.png
deleted file mode 100644
index a260e024cb..0000000000
Binary files a/website/docs/modeling/model_design/images/computed_column/cc_3.png and /dev/null differ
diff --git a/website/docs/modeling/model_design/images/computed_column/cc_4.png b/website/docs/modeling/model_design/images/computed_column/cc_4.png
deleted file mode 100644
index 5f12cd3f80..0000000000
Binary files a/website/docs/modeling/model_design/images/computed_column/cc_4.png and /dev/null differ
diff --git a/website/docs/modeling/model_design/images/computed_column/cc_5.png b/website/docs/modeling/model_design/images/computed_column/cc_5.png
deleted file mode 100644
index 1feba27bb6..0000000000
Binary files a/website/docs/modeling/model_design/images/computed_column/cc_5.png and /dev/null differ
diff --git a/website/docs/modeling/model_design/images/computed_column/cc_6.png b/website/docs/modeling/model_design/images/computed_column/cc_6.png
deleted file mode 100644
index ec280e72be..0000000000
Binary files a/website/docs/modeling/model_design/images/computed_column/cc_6.png and /dev/null differ
diff --git a/website/docs/modeling/model_design/images/computed_column/cc_8.png b/website/docs/modeling/model_design/images/computed_column/cc_8.png
deleted file mode 100644
index 017bb56600..0000000000
Binary files a/website/docs/modeling/model_design/images/computed_column/cc_8.png and /dev/null differ
diff --git a/website/docs/modeling/model_design/images/count_distinct/cd_measures.png b/website/docs/modeling/model_design/images/count_distinct/cd_measures.png
deleted file mode 100644
index b583c4309c..0000000000
Binary files a/website/docs/modeling/model_design/images/count_distinct/cd_measures.png and /dev/null differ
diff --git a/website/docs/modeling/model_design/images/count_distinct/cd_measures_add.1.png b/website/docs/modeling/model_design/images/count_distinct/cd_measures_add.1.png
deleted file mode 100644
index b8c0cfbcaf..0000000000
Binary files a/website/docs/modeling/model_design/images/count_distinct/cd_measures_add.1.png and /dev/null differ
diff --git a/website/docs/modeling/model_design/images/count_distinct/cd_measures_add.2.png b/website/docs/modeling/model_design/images/count_distinct/cd_measures_add.2.png
deleted file mode 100644
index e190b7535e..0000000000
Binary files a/website/docs/modeling/model_design/images/count_distinct/cd_measures_add.2.png and /dev/null differ
diff --git a/website/docs/modeling/model_design/images/count_distinct/cd_measures_add.3.png b/website/docs/modeling/model_design/images/count_distinct/cd_measures_add.3.png
deleted file mode 100644
index 83a6431800..0000000000
Binary files a/website/docs/modeling/model_design/images/count_distinct/cd_measures_add.3.png and /dev/null differ
diff --git a/website/docs/modeling/model_design/images/count_distinct/cd_measures_add.4.png b/website/docs/modeling/model_design/images/count_distinct/cd_measures_add.4.png
deleted file mode 100644
index 7d306a51a7..0000000000
Binary files a/website/docs/modeling/model_design/images/count_distinct/cd_measures_add.4.png and /dev/null differ
diff --git a/website/docs/modeling/model_design/images/count_distinct/cd_measures_add.5.png b/website/docs/modeling/model_design/images/count_distinct/cd_measures_add.5.png
deleted file mode 100644
index 72a19cb9c4..0000000000
Binary files a/website/docs/modeling/model_design/images/count_distinct/cd_measures_add.5.png and /dev/null differ
diff --git a/website/docs/modeling/model_design/images/count_distinct/cd_measures_add.7.png b/website/docs/modeling/model_design/images/count_distinct/cd_measures_add.7.png
deleted file mode 100644
index 8acb551cf0..0000000000
Binary files a/website/docs/modeling/model_design/images/count_distinct/cd_measures_add.7.png and /dev/null differ
diff --git a/website/docs/modeling/model_design/images/count_distinct/cd_measures_add.8.png b/website/docs/modeling/model_design/images/count_distinct/cd_measures_add.8.png
deleted file mode 100644
index f5e69a1e0f..0000000000
Binary files a/website/docs/modeling/model_design/images/count_distinct/cd_measures_add.8.png and /dev/null differ
diff --git a/website/docs/modeling/model_design/images/count_distinct/cd_measures_add.9.png b/website/docs/modeling/model_design/images/count_distinct/cd_measures_add.9.png
deleted file mode 100644
index 6899b18711..0000000000
Binary files a/website/docs/modeling/model_design/images/count_distinct/cd_measures_add.9.png and /dev/null differ
diff --git a/website/docs/modeling/model_design/images/count_distinct/cd_measures_add.png b/website/docs/modeling/model_design/images/count_distinct/cd_measures_add.png
deleted file mode 100644
index 6791944b4a..0000000000
Binary files a/website/docs/modeling/model_design/images/count_distinct/cd_measures_add.png and /dev/null differ
diff --git a/website/docs/modeling/model_design/images/count_distinct/cd_measures_add_precisely.png b/website/docs/modeling/model_design/images/count_distinct/cd_measures_add_precisely.png
deleted file mode 100644
index 4fc3db6595..0000000000
Binary files a/website/docs/modeling/model_design/images/count_distinct/cd_measures_add_precisely.png and /dev/null differ
diff --git a/website/docs/modeling/model_design/images/count_distinct/cd_measures_edit.png b/website/docs/modeling/model_design/images/count_distinct/cd_measures_edit.png
deleted file mode 100644
index e190b7535e..0000000000
Binary files a/website/docs/modeling/model_design/images/count_distinct/cd_measures_edit.png and /dev/null differ
diff --git a/website/docs/modeling/model_design/images/count_distinct/cd_meausres_add.6.png b/website/docs/modeling/model_design/images/count_distinct/cd_meausres_add.6.png
deleted file mode 100644
index 473fab2a1b..0000000000
Binary files a/website/docs/modeling/model_design/images/count_distinct/cd_meausres_add.6.png and /dev/null differ
diff --git a/website/docs/modeling/model_design/images/count_distinct/wd_datasample.png b/website/docs/modeling/model_design/images/count_distinct/wd_datasample.png
deleted file mode 100644
index 19e45dc4b2..0000000000
Binary files a/website/docs/modeling/model_design/images/count_distinct/wd_datasample.png and /dev/null differ
diff --git a/website/docs/modeling/model_design/images/import_sql.en.jpg b/website/docs/modeling/model_design/images/import_sql.en.jpg
deleted file mode 100644
index de7b5c69f9..0000000000
Binary files a/website/docs/modeling/model_design/images/import_sql.en.jpg and /dev/null differ
diff --git a/website/docs/modeling/model_design/images/mdc/intro_mdc.png b/website/docs/modeling/model_design/images/mdc/intro_mdc.png
deleted file mode 100644
index 1ab214937e..0000000000
Binary files a/website/docs/modeling/model_design/images/mdc/intro_mdc.png and /dev/null differ
diff --git a/website/docs/modeling/model_design/images/mdc/single_mdc.png b/website/docs/modeling/model_design/images/mdc/single_mdc.png
deleted file mode 100644
index 0a575c2640..0000000000
Binary files a/website/docs/modeling/model_design/images/mdc/single_mdc.png and /dev/null differ
diff --git a/website/docs/modeling/model_design/images/mdc/total_mdc.png b/website/docs/modeling/model_design/images/mdc/total_mdc.png
deleted file mode 100644
index 37ee898e6e..0000000000
Binary files a/website/docs/modeling/model_design/images/mdc/total_mdc.png and /dev/null differ
diff --git a/website/docs/modeling/model_design/images/model_check/24_model_diagnose_1.png b/website/docs/modeling/model_design/images/model_check/24_model_diagnose_1.png
deleted file mode 100644
index 47d6afe09e..0000000000
Binary files a/website/docs/modeling/model_design/images/model_check/24_model_diagnose_1.png and /dev/null differ
diff --git a/website/docs/modeling/model_design/images/model_check/24_model_diagnose_2.png b/website/docs/modeling/model_design/images/model_check/24_model_diagnose_2.png
deleted file mode 100644
index 584e579ecc..0000000000
Binary files a/website/docs/modeling/model_design/images/model_check/24_model_diagnose_2.png and /dev/null differ
diff --git a/website/docs/modeling/model_design/images/model_check/24_model_diagnose_4.png b/website/docs/modeling/model_design/images/model_check/24_model_diagnose_4.png
deleted file mode 100644
index 46c6d36cef..0000000000
Binary files a/website/docs/modeling/model_design/images/model_check/24_model_diagnose_4.png and /dev/null differ
diff --git a/website/docs/modeling/model_design/images/model_check/24_model_diagnose_6.png b/website/docs/modeling/model_design/images/model_check/24_model_diagnose_6.png
deleted file mode 100644
index 6e02793523..0000000000
Binary files a/website/docs/modeling/model_design/images/model_check/24_model_diagnose_6.png and /dev/null differ
diff --git a/website/docs/modeling/model_design/images/model_check/25_model_check.png b/website/docs/modeling/model_design/images/model_check/25_model_check.png
deleted file mode 100644
index 86426f368d..0000000000
Binary files a/website/docs/modeling/model_design/images/model_check/25_model_check.png and /dev/null differ
diff --git a/website/docs/modeling/model_design/images/model_check/25_model_save.png b/website/docs/modeling/model_design/images/model_check/25_model_save.png
deleted file mode 100644
index b1cc0f8647..0000000000
Binary files a/website/docs/modeling/model_design/images/model_check/25_model_save.png and /dev/null differ
diff --git a/website/docs/modeling/model_design/images/model_design/add_table.png b/website/docs/modeling/model_design/images/model_design/add_table.png
deleted file mode 100644
index 69c4ebd63b..0000000000
Binary files a/website/docs/modeling/model_design/images/model_design/add_table.png and /dev/null differ
diff --git a/website/docs/modeling/model_design/images/model_design/dimension.png b/website/docs/modeling/model_design/images/model_design/dimension.png
deleted file mode 100644
index bb16202ae3..0000000000
Binary files a/website/docs/modeling/model_design/images/model_design/dimension.png and /dev/null differ
diff --git a/website/docs/modeling/model_design/images/model_design/dimension_2.png b/website/docs/modeling/model_design/images/model_design/dimension_2.png
deleted file mode 100644
index 49fe9e8daa..0000000000
Binary files a/website/docs/modeling/model_design/images/model_design/dimension_2.png and /dev/null differ
diff --git a/website/docs/modeling/model_design/images/model_design/join.png b/website/docs/modeling/model_design/images/model_design/join.png
deleted file mode 100644
index 5c4fbfd9e6..0000000000
Binary files a/website/docs/modeling/model_design/images/model_design/join.png and /dev/null differ
diff --git a/website/docs/modeling/model_design/images/model_design/measure.png b/website/docs/modeling/model_design/images/model_design/measure.png
deleted file mode 100644
index 7a4110a936..0000000000
Binary files a/website/docs/modeling/model_design/images/model_design/measure.png and /dev/null differ
diff --git a/website/docs/modeling/model_design/images/model_design/measure_2.png b/website/docs/modeling/model_design/images/model_design/measure_2.png
deleted file mode 100644
index eb35cfd448..0000000000
Binary files a/website/docs/modeling/model_design/images/model_design/measure_2.png and /dev/null differ
diff --git a/website/docs/modeling/model_design/images/model_design/measure_3.png b/website/docs/modeling/model_design/images/model_design/measure_3.png
deleted file mode 100644
index 925895a4c6..0000000000
Binary files a/website/docs/modeling/model_design/images/model_design/measure_3.png and /dev/null differ
diff --git a/website/docs/modeling/model_design/images/model_design/model_edit_page.png b/website/docs/modeling/model_design/images/model_design/model_edit_page.png
deleted file mode 100644
index 72a4b4520f..0000000000
Binary files a/website/docs/modeling/model_design/images/model_design/model_edit_page.png and /dev/null differ
diff --git a/website/docs/modeling/model_design/images/model_design/model_filter_condition.en.png b/website/docs/modeling/model_design/images/model_design/model_filter_condition.en.png
deleted file mode 100644
index 61a5d63688..0000000000
Binary files a/website/docs/modeling/model_design/images/model_design/model_filter_condition.en.png and /dev/null differ
diff --git a/website/docs/modeling/model_design/images/model_design/model_result.png b/website/docs/modeling/model_design/images/model_design/model_result.png
deleted file mode 100644
index adf9efec54..0000000000
Binary files a/website/docs/modeling/model_design/images/model_design/model_result.png and /dev/null differ
diff --git a/website/docs/modeling/model_design/images/model_design/partition_en.png b/website/docs/modeling/model_design/images/model_design/partition_en.png
deleted file mode 100644
index 4f70f6a72b..0000000000
Binary files a/website/docs/modeling/model_design/images/model_design/partition_en.png and /dev/null differ
diff --git a/website/docs/modeling/model_design/images/model_design/sync_hive_comment_en.png b/website/docs/modeling/model_design/images/model_design/sync_hive_comment_en.png
deleted file mode 100644
index 8e0ab1a8b4..0000000000
Binary files a/website/docs/modeling/model_design/images/model_design/sync_hive_comment_en.png and /dev/null differ
diff --git a/website/docs/modeling/model_design/images/percentile_approximate/1.en.png b/website/docs/modeling/model_design/images/percentile_approximate/1.en.png
deleted file mode 100644
index 7a7ee394a5..0000000000
Binary files a/website/docs/modeling/model_design/images/percentile_approximate/1.en.png and /dev/null differ
diff --git a/website/docs/modeling/model_design/images/percentile_approximate/cube_query.en.png b/website/docs/modeling/model_design/images/percentile_approximate/cube_query.en.png
deleted file mode 100644
index 7377b3114f..0000000000
Binary files a/website/docs/modeling/model_design/images/percentile_approximate/cube_query.en.png and /dev/null differ
diff --git a/website/docs/modeling/model_design/images/percentile_approximate/return_type.en.png b/website/docs/modeling/model_design/images/percentile_approximate/return_type.en.png
deleted file mode 100644
index ef1567ebcd..0000000000
Binary files a/website/docs/modeling/model_design/images/percentile_approximate/return_type.en.png and /dev/null differ
diff --git a/website/docs/modeling/model_design/images/review_model.png b/website/docs/modeling/model_design/images/review_model.png
deleted file mode 100644
index 9ab12c2374..0000000000
Binary files a/website/docs/modeling/model_design/images/review_model.png and /dev/null differ
diff --git a/website/docs/modeling/model_design/images/table_index/index_1.png b/website/docs/modeling/model_design/images/table_index/index_1.png
deleted file mode 100644
index f9ea65c760..0000000000
Binary files a/website/docs/modeling/model_design/images/table_index/index_1.png and /dev/null differ
diff --git a/website/docs/modeling/model_design/images/table_index/table_index_1.png b/website/docs/modeling/model_design/images/table_index/table_index_1.png
deleted file mode 100644
index c4e9b6d662..0000000000
Binary files a/website/docs/modeling/model_design/images/table_index/table_index_1.png and /dev/null differ
diff --git a/website/docs/modeling/model_design/images/table_index/table_index_2.png b/website/docs/modeling/model_design/images/table_index/table_index_2.png
deleted file mode 100644
index 5c642138a4..0000000000
Binary files a/website/docs/modeling/model_design/images/table_index/table_index_2.png and /dev/null differ
diff --git a/website/docs/modeling/model_design/images/table_index/table_index_3.png b/website/docs/modeling/model_design/images/table_index/table_index_3.png
deleted file mode 100644
index ceb17808c2..0000000000
Binary files a/website/docs/modeling/model_design/images/table_index/table_index_3.png and /dev/null differ
diff --git a/website/docs/modeling/model_design/images/table_index/table_index_disable.png b/website/docs/modeling/model_design/images/table_index/table_index_disable.png
deleted file mode 100644
index 867642b278..0000000000
Binary files a/website/docs/modeling/model_design/images/table_index/table_index_disable.png and /dev/null differ
diff --git a/website/docs/modeling/model_design/images/table_index/table_index_enable.png b/website/docs/modeling/model_design/images/table_index/table_index_enable.png
deleted file mode 100644
index 3157180d0d..0000000000
Binary files a/website/docs/modeling/model_design/images/table_index/table_index_enable.png and /dev/null differ
diff --git a/website/docs/modeling/model_design/images/table_index/table_index_encode.png b/website/docs/modeling/model_design/images/table_index/table_index_encode.png
deleted file mode 100644
index 4468b92cd8..0000000000
Binary files a/website/docs/modeling/model_design/images/table_index/table_index_encode.png and /dev/null differ
diff --git a/website/docs/modeling/model_design/images/table_index/table_index_index.png b/website/docs/modeling/model_design/images/table_index/table_index_index.png
deleted file mode 100644
index 76313978dd..0000000000
Binary files a/website/docs/modeling/model_design/images/table_index/table_index_index.png and /dev/null differ
diff --git a/website/docs/modeling/model_design/images/table_index/table_index_sortby.png b/website/docs/modeling/model_design/images/table_index/table_index_sortby.png
deleted file mode 100644
index c7054dd917..0000000000
Binary files a/website/docs/modeling/model_design/images/table_index/table_index_sortby.png and /dev/null differ
diff --git a/website/docs/modeling/model_design/images/topN_1.png b/website/docs/modeling/model_design/images/topN_1.png
deleted file mode 100644
index b9da18aa7f..0000000000
Binary files a/website/docs/modeling/model_design/images/topN_1.png and /dev/null differ
diff --git a/website/docs/modeling/model_design/images/topN_measure_edit.png b/website/docs/modeling/model_design/images/topN_measure_edit.png
deleted file mode 100644
index 28db73fe69..0000000000
Binary files a/website/docs/modeling/model_design/images/topN_measure_edit.png and /dev/null differ
diff --git a/website/docs/modeling/model_design/intro.md b/website/docs/modeling/model_design/intro.md
deleted file mode 100644
index 9c94754b8d..0000000000
--- a/website/docs/modeling/model_design/intro.md
+++ /dev/null
@@ -1,17 +0,0 @@
----
-title: Advanced model design
-language: en
-sidebar_label: Advanced model design
-pagination_label: Advanced model design
-toc_min_heading_level: 2
-toc_max_heading_level: 6
-pagination_prev: null
-pagination_next: null
-keywords:
- - advanced model design
-draft: false
-last_update:
- date: 09/13/2022
----
-
-In this chapter, we take the Kylin built-in dataset as an example, to show you how to design a data model.
diff --git a/website/docs/modeling/model_design/table_index.md b/website/docs/modeling/model_design/table_index.md
deleted file mode 100644
index 849034e60b..0000000000
--- a/website/docs/modeling/model_design/table_index.md
+++ /dev/null
@@ -1,68 +0,0 @@
----
-title: Table Index
-language: en
-sidebar_label: Table Index
-pagination_label: Table Index
-toc_min_heading_level: 2
-toc_max_heading_level: 6
-pagination_prev: null
-pagination_next: null
-keywords:
- - table index
-draft: false
-last_update:
- date: 08/19/2022
----
-
-
-Kylin provides table index function to support querying on raw data. To create a table index, similar to a table, you need to select fields that are often queried on the transaction level. For customized queries, this product leverages well-designed aggregate index to efficiently process them. For queries not within the scope, **Query Pushdown** plays a supplementary role.
-
-
-### Create Table Index
-
-In the **Data Asset -> Model** page, click the model name，then click **Index** and click **Table Index** button under **+ Index** in the **Index Overview** tab or **+**(Add Table Index) button in the **Table Index** tab to enter the **Add Table Index** page. You can edit the table index in the pop-up window shown below and put the required column in a table index.
-
-
-
-The fields are defined as follows:
-
-1. **Column Name**
-
- All the columns defined as **dimension** or referenced by **measure** in the model.
-
-2. **Cardinality**
-
- Cardinality of the column.
-
-3. **Order**
-
- Reasonably setting a certain column as SortBy could substantially increase query efficiency of table index. When you select the column ABC, you can click the arrows in **Order** to adjust the column order.
-
- Those top columns in the order of SortBy list will provide more efficiency benefits they are used as a filter condition in queries. We suggest you arrange SortBy columns in order of frequency they will be used as a filter condition. Do not set too many SortBy columns because tail columns do to little help performance.
-
-4. **ShardBy**
-
- If you want to set a column as the ShardBy column, select it and click the icon below the **ShardBy**.
-
- When configuring table index you can set one column as ShardBy or simply zero. If you set a certain column as the ShardBy column, the raw data will be sharded according to values of that column. If you don't explicitly specify an ShardBy column, sharding will be done taking into consideration all the column values.
-
- Select an appropriate ShardBy column could distribute raw data into multiple shards, which can increase concurrency, and achieve a better query performance. We suggest you choose columns with relatively large cardinality as the ShardBy column to prevent heterogeneous data distribution.
-
-
-You can add multiple table indices. After building the table index and loading data, you can query for corresponding raw data.
-
-
-
-### View Table Index
-
-In the **Data Asset -> Model** page, click the model name and click **Index**. Under the **Index Overview** tab, you can view the details of table index.
-
-You can click the icon beside the status and see the details in View Index Details, such as the columns in the table index, the order, and the shardby column. For the custom table index, you can click the edit button to edit it, or click the delete button to delete it. You can also delete the recommended table index, but can not edit it.
-
-If you need to add base indexes when there is no base index or when a base index is missing. Click **+ Index** to add the base indexes in the drop-down box.
-
-
-
-
-
-For more information about the Index List, such as the index status, please refer to the [Aggregate Index](aggregation_group.md) section.
diff --git a/website/docs/modeling/model_management.md b/website/docs/modeling/model_management.md
deleted file mode 100644
index 43c79ca0a9..0000000000
--- a/website/docs/modeling/model_management.md
+++ /dev/null
@@ -1,137 +0,0 @@
----
-title: Model Management
-language: en
-sidebar_label: Model Management
-pagination_label: Model Management
-toc_min_heading_level: 2
-toc_max_heading_level: 6
-pagination_prev: null
-pagination_next: null
-keywords:
- - model management
- - model operations
-draft: false
-last_update:
- date: 09/13/2022
----
-
-
-Model design refers to build the star model or snowflake model based on data table and multidimensional modeling theory. The main contents of model design are as follows:
-
-- Define Fact Table and Dimension Table
-- Define the Association Relationship between Tables
-- Define Dimension and Measurement
-
-### <span id="model">Model List</span>
-
- You can create and design models manually. Below are the main content of model list:
-
-1. Log in to Web UI, switch to a project.
-
-2. Navigate to **Data Asset -> Model** page, where models are shown in a list. The picture below is the index group list:
-
- 
-
- **Fields Explanation:**
-
- - **Model Name**: Model's name.
-
- - **Status**: There are four statuses.
- - *ONLINE* indicates this model is online and is able to answer queries.
- - *OFFLINE* indicates this model is offline and not available to answer queries. We recommend using offline when you need to edit the model.
- - *BROKEN* indicates this model is broken and not available. Mostly happens when the schemas of related source tables have changed, for instance, a related source table is deleted.
- - *WARNING* indicates this model is warning and can only server parts of queries. Mostly happens when the segments exist holes or indexes are waiting to build.
- - **Last Updated Time**: The lastest time to update model.
-
- - **More Actions**: The **More Actions** button will appear when you are hovering on model name area, please refer to [Model Operations](#operation) for details.
-
- - **Owner**: The user who created this model.
-
- - **Description**: Model description.
-
- - **Fact Table**: The fact table of this model.
-
- - **Types**: Model types, which include *Batch Model*
-
- - **Usage**: Hit count by SQL statements in the last 30 days. Update every 30 minutes.
-
- - **Rows**: The rows of loaded data in this model.
-
- - **Storage**: The storage size of loaded data in this model, which combines the storage size of all Segments data.
-
- > Tip: When the tiered storage is turned on, the total storage size of the data loaded into the tiered storage (ClickHouse) will be displayed.
-
- - **Expansion Rate**: The ratio of the storage size of the built data to the storage size of the corresponding source table data under the model. Expansion Rate = Storage Size / Source Table Size.
-
-
- > Notice: The expansion rate won't show if the storage size is less than 1GB.
-
- - **Index Amount**: The amount of indexes in this model.
-
-### <span id="operation">Model Operation</span>
-
-You are only allowed to operate on models. You can hover on the right most column **Actions** of the model list to get the action names. Specific actions are listed below:
-
-- **Edit**: Click on the pencil shape button, enter into the model editing page.
-
-- **Build Index**: Loads data for models. You can choose the data range in the pop-up window.
-
-- **Model Partition**: Set partition column for the model.
-
-- **Export Model**: Export single model metadata.
-
- > **Note**: Since the locked indexes will be deleted after the new indexes have been built, the exported model metadata will not include the locked index.
-
-- **Export TDS**: Export TDS file of the model .
-
-- **Rename**: Renames the model.
-
-- **Clone**: Clones an identical model. You can give a new name for this new model. The new model has the same fact table, dimension tables, join relationship, dimensions, measures, computed columns, date partition column, aggregate indexes, table indexes, etc. as the origin model. But the new model does not have data, you need to load data for this cloned model manually.
-
- > **Note**: Since the locked indexes will be deleted after the new indexes have been built, the cloned model will not include the locked index.
-
-- **Change Owner**：Change model owner. Only system administrators and project administrators have the authority to modify model owner.
-
-- **Delete**: Deletes the model, remove the loaded data at the same time.
-
-- **Purge**: Purges all loaded data in this model.
-
-- **Offline**: Makes a *Online / Warning* model offline. An offline model cannot answer any queries.
-
-- **Online**: Makes a *Offline* model online. An online model should be able to answer related queries.
-
-> **Note:** If the model is in *BROKEN* status, only the **delete** operation is allowed.
-
-
-### <span id="more">Model Details</span>
-
-Models contain Segments and indexes. You can click model name to unfold the detailed information, as shown below:
-
-
-
-- **Overview**: Check Overview details, please refer to [Model Overview](#overview) for more.
-- **Data Features**: Check data features.
-- **Segment**: Check Segment details, please refer to [Segment Operation and Settings](load_data/segment_operation_settings/intro.md) for more.
-- **Index**: Review the model indexes.
- - **Index Overview**: Check index overview.
- - **Aggregate Group**: Add or check defined aggregate indexes, please refer to [Aggregate Index](model_design/aggregation_group.md) for more details.
- - **Table Index**: Add or check defined table indexes, please refer to [Table Index](model_design/table_index.md) for more details.
-- **Developers**: Check information for developers.
- - **JSON**: Kylin describes the information of models (index groups) in `JSON` format, such as design, dimensions, measures, etc.
- - **SQL**: The SQL statement consists of related information about tables and columns in the model, such as the join conditions between the tables.
-
-### <span id="overview">Model Overview</span>
-
-After expanding the model information, you can see the model overview page, which will help you to quickly get the model information.
-
-
-
-On this page, you can view the ER diagram of the model.
-
-
-
-What's more, you can view the dimensions and measures information contained in the model.
-
-
-
-
diff --git a/website/docs/operations/job-monitoring/intro.md b/website/docs/operations/job-monitoring/intro.md
index 1c541497e6..bd6bb68286 100644
--- a/website/docs/operations/job-monitoring/intro.md
+++ b/website/docs/operations/job-monitoring/intro.md
@@ -22,5 +22,5 @@ For a clearer understanding of how to trigger a job, we recommend reading the fo
- [Data Source](../../datasource/intro.md)
- [SQL Query](../../query/insight)
-- [Model](../../modeling/intro.md)
+- [Model](../../model/intro.md)
diff --git a/website/docs/operations/job-monitoring/job_concept_settings.md b/website/docs/operations/job-monitoring/job_concept_settings.md
index d3918d450d..186b72dd6a 100644
--- a/website/docs/operations/job-monitoring/job_concept_settings.md
+++ b/website/docs/operations/job-monitoring/job_concept_settings.md
@@ -40,7 +40,7 @@ You can view the job details in the navigation bar **Monitor -> Job**. As shown
- In the navigation bar **Data Asset -> Model**, creating new models will trigger this job.
- > **Tips**: For details on how to build an index, please see [Aggregate Index](../../modeling/model_design/aggregation_group.md).
+ > **Tips**: For details on how to build an index, please see [Aggregate Index](../../model/model_design/aggregation_group.md).
2. **Load Data**: Job for incrementally loading data on existing models/indices.
@@ -48,7 +48,7 @@ You can view the job details in the navigation bar **Monitor -> Job**. As shown
- In the navigation bar **Data Asset -> Model**, loading data within a time range of the model will trigger the job.
-3. **Merge Data**: When the system detects a sufficient number of segments, it will automatically trigger the job of merging data. You can set the parameters of merging segments in the navigation bar **Setting -> Segment Settings**. For detailed message, you can refer to [Segment Operation and Settings](../../modeling/load_data/segment_operation_settings/intro.md).
+3. **Merge Data**: When the system detects a sufficient number of segments, it will automatically trigger the job of merging data. You can set the parameters of merging segments in the navigation bar **Setting -> Segment Settings**. For detailed message, you can refer to [Segment Operation and Settings](../../model/load_data/segment_operation_settings/intro.md).
4. **Refresh Data**: Job for refreshing segments.
diff --git a/website/docs/operations/job-monitoring/job_exception_resolve.md b/website/docs/operations/job-monitoring/job_exception_resolve.md
index a3905699f2..1ff4a8e109 100644
--- a/website/docs/operations/job-monitoring/job_exception_resolve.md
+++ b/website/docs/operations/job-monitoring/job_exception_resolve.md
@@ -28,7 +28,7 @@ Various problems may occur during the execution of building jobs which cause the
1. Modify the time format of the model time partition column to be consistent with the actual time format in the data source:
- Please refer to [Design a Data Model](../../modeling/manual_modeling.md#step-4-save-the-model-and-set-the-loading-method) *Step 4. Save the model and set the loading method* modify the time format of the model time partition column。
+ Please refer to [Design a Data Model](../../model/manual_modeling.md#step-4-save-the-model-and-set-the-loading-method) *Step 4. Save the model and set the loading method* modify the time format of the model time partition column。
2. 2. If you insist on using this format, you can choose to disable checking the time partition column by modifying the system parameter in `kylin.properties` to `kylin.engine.check-partition-col-enabled=false`.
Notice: Although this method can bypass the time format verification here, it may cause other problems. Please use it with caution.
diff --git a/website/docs/overview.md b/website/docs/overview.md
index 115e6a4c93..ec94c43b32 100644
--- a/website/docs/overview.md
+++ b/website/docs/overview.md
@@ -37,7 +37,7 @@ Kylin now support internal table, which is designed for flexible query and lakeh
With recommendation engine, you don't have to be an expert of modeling. Kylin now can auto modeling and optimizing indexes from you query history.
You can also create model by importing sql text.
->More details, please refer to [Auto Modeling](modeling/auto_modeling/intro.md) and [Index Optimization](modeling/auto_modeling/optimize_index/intro.md)
+>More details, please refer to [Auto Modeling](model/rec/sql_modeling.md) and [Index Optimization](model/rec/optimize_index/intro.md)
### 3. Native Compute Engine
@@ -50,4 +50,4 @@ Which can bring 3~4x performance improvement compared with vanilla spark. Both m
Kylin now support Apache Kafka as streaming data source of model building. Users can create a fusion model to implement streaming-batch hybrid analysis.
-## Other Optimizations and Improvements
\ No newline at end of file
+## Other Optimizations and Improvements
diff --git a/website/docs/query/query_history.md b/website/docs/query/history/query_history.md
similarity index 94%
rename from website/docs/query/query_history.md
rename to website/docs/query/history/query_history.md
index cfb53dc109..44ef5dc4b9 100644
--- a/website/docs/query/query_history.md
+++ b/website/docs/query/history/query_history.md
@@ -1,8 +1,8 @@
---
-title: Query History
+title: Guide
language: en
-sidebar_label: Query History
-pagination_label: Query History
+sidebar_label: Guide
+pagination_label: Guide
toc_min_heading_level: 2
toc_max_heading_level: 6
pagination_prev: null
@@ -55,7 +55,7 @@ This page saves the basic information of queries, such as query time, SQL statem
Please enter the query history page by clicking **Query -> History** in the navigator bar, after executing the above SQL statements, you will see the following content.
-
+
Each line in the picture is a query history record. The meaning of the columns are as follows:
@@ -89,7 +89,7 @@ Each line in the picture is a query history record. The meaning of the columns a
When you click on the icon to the left of a query, the execution details of the query will be displayed, as shown below:
-
+
On the left is the SQL statement, you can copy and paste and then query. The fields on the right have the following meanings:
@@ -105,7 +105,7 @@ On the left is the SQL statement, you can copy and paste and then query. The fie
When you click on the error details to the right of a query failure, the exception details of that query are displayed，as shown below:
-
+
> **Note**: When SQL does not wrap, only the first 100 lines of SQL can be seen in the query execution details. When SQL wraps, the details display the first 2000 characters. You can click the copy button in the upper right corner of the SQL statement box to copy the complete SQL statement.
@@ -132,10 +132,10 @@ kylin.query.queryhistory.survival-time-threshold=30
### Export History Records
The query history can be exported as a csv file, as shown in the figure below, the export result is the filtered data.
-
+
You can also export the sql in the query history as a text file separately, as shown in the figure below, the export result is also the filtered data.
-
+
Query history support configuration export upper limit, the default value is 100000.
```shell
diff --git a/website/docs/deployment/on-premises/rdbms_metastore/query_history_fields.md b/website/docs/query/history/query_history_fields.md
similarity index 98%
rename from website/docs/deployment/on-premises/rdbms_metastore/query_history_fields.md
rename to website/docs/query/history/query_history_fields.md
index 936ef90abe..5497e1c041 100644
--- a/website/docs/deployment/on-premises/rdbms_metastore/query_history_fields.md
+++ b/website/docs/query/history/query_history_fields.md
@@ -1,14 +1,14 @@
---
-title: Query History
+title: Table Schema
language: en
-sidebar_label: Query History
-pagination_label: Query History
+sidebar_label: Table Schema
+pagination_label: Table Schema
toc_min_heading_level: 2
toc_max_heading_level: 6
pagination_prev: null
pagination_next: null
keywords:
- - query history
+ - table schema
draft: false
last_update:
date: 09/13/2022
diff --git a/website/docs/quickstart/intro.md b/website/docs/quickstart/intro.md
index e3a0e62d6c..daf793e107 100644
--- a/website/docs/quickstart/intro.md
+++ b/website/docs/quickstart/intro.md
@@ -72,23 +72,7 @@ docker run -d \
For production environments, we strongly recommend setting up a dedicated metastore using either PostgreSQL or MySQL to ensure reliability.
-6. Install InfluxDB(**Deprecated**).
-
- Kylin utilizes InfluxDB to store system monitoring data. This step is optional, but highly recommended for production environments to leverage monitoring capabilities.
-
- ```sh
- # download influxdb
- $KYLIN_HOME/sbin/download-influxdb.sh
-
- cd $KYLIN_HOME/influxdb
-
- # install influxdb
- rpm -ivh influxdb-1.6.5.x86_64.rpm
- ```
-
- For more details, please refer to [Use InfluxDB as Time-Series Database](../operations/system-monitoring/influxdb/influxdb.md).
-
-7. Create a working directory on HDFS and grant permissions.
+6. Create a working directory on HDFS and grant permissions.
The default working directory is `/kylin`. Also ensure the Linux account has access to its home directory on HDFS. Meanwhile, create directory `/kylin/spark-history` to store the spark log files.
diff --git a/website/docs/quickstart/tutorial.md b/website/docs/quickstart/tutorial.md
index 18cfecb679..a3278e2573 100644
--- a/website/docs/quickstart/tutorial.md
+++ b/website/docs/quickstart/tutorial.md
@@ -86,11 +86,11 @@ Model design is one of the core functions of Kylin. Good model design can help a
2. **Method of Model Design**
- Please create a model in the **Data Asset -> Model** interface and enter the model editing interface to visually complete the creation of the multidimensional model. The specific method of model design will be described in detail in the [Model Design Basics](../modeling/intro.md) section. Here is a brief introduction to the following steps:
+ Please create a model in the **Data Asset -> Model** interface and enter the model editing interface to visually complete the creation of the multidimensional model. The specific method of model design will be described in detail in the [Model Design Basics](../model/intro.md) section. Here is a brief introduction to the following steps:
- Design Model: Select the source table according to the business logic and set the association between the tables. Then set the fact table and dimension tables.
- Add Dimensions: Identify dimensions from table columns for business analysis.
- - Add Measures: Identify metrics and their aggregate functions for business analytics. You can see the detailed methods in the [Measures](../modeling/model_design/measure_design/intro.md) section.
+ - Add Measures: Identify metrics and their aggregate functions for business analytics. You can see the detailed methods in the [Measures](../model/model_design/measures/intro.md) section.
As shown in the following diagram, we built the model using the source tables in the SSB dataset.
@@ -106,8 +106,8 @@ After the model is created, you need to define the index in the model; this shou
Not every dimension combination is needed for business analysis. In this case, pre-calculating all the dimension combinations will bring a large workload and can result in a long indexing time and a large data storage space. We can improve this by adding aggregate index and table index.
- - Aggregate Index: A group of dimension combinations customized to a particular business analysis. For example, an online store analyst needs to analyze the purchasing power of male and female customers in different cities, and the dimension combination in the index is `[city, customer gender]`. At this time, other dimensions are not needed in the index. If you do not need to analyze the product category, then there is no need to include it in any index. You can find out more in the [ [...]
- - Table Index: Table index supports efficient querying of detailed data records. For example, an online store analyst needs to query the detailed order data, they can add `[OrderKey, OrderDate, PartKey, CustomerKey, OrderQuantity, OrderAmount]` in a table index. After building the index and loading the data, they can query the detailed data records efficiently. You can find out more in the [Table Index](../modeling/model_design/table_index.md) section.
+ - Aggregate Index: A group of dimension combinations customized to a particular business analysis. For example, an online store analyst needs to analyze the purchasing power of male and female customers in different cities, and the dimension combination in the index is `[city, customer gender]`. At this time, other dimensions are not needed in the index. If you do not need to analyze the product category, then there is no need to include it in any index. You can find out more in the [ [...]
+ - Table Index: Table index supports efficient querying of detailed data records. For example, an online store analyst needs to query the detailed order data, they can add `[OrderKey, OrderDate, PartKey, CustomerKey, OrderQuantity, OrderAmount]` in a table index. After building the index and loading the data, they can query the detailed data records efficiently. You can find out more in the [Table Index](../model/model_design/table_index.md) section.
2. **How to design an index**
- Edit aggregate group: On the left navigation bar, click **Data Asset -> Model**. In the **Model List** page, click one model to enter the specific model page, click **Index**. In the **Index Overview** tab, click **+ Index -> Aggregate Group**. The diagram below shows an aggregation group in the built-in demo. There are basically four concepts in an aggregation group in order to control the combination of dimensions:
@@ -124,13 +124,13 @@ After the model is created, you need to define the index in the model; this shou
### <span id="loaddata">Load Data</span>
-Kylin applies pre-calculation technology to achieve sub-second query response time in the big data era. After creating the model and editing the index, you need to load the data for the model. The process of loading data is also the pre-calculation process for the pre-defined index. Models that do not have data loaded cannot serve queries. You can find out more about how to load data from the [Load Data](../modeling/load_data/intro.md) section.
+Kylin applies pre-calculation technology to achieve sub-second query response time in the big data era. After creating the model and editing the index, you need to load the data for the model. The process of loading data is also the pre-calculation process for the pre-defined index. Models that do not have data loaded cannot serve queries. You can find out more about how to load data from the [Load Data](../model/load_data/intro.md) section.
1. **Principle of Loading Data**
- Set time partition column: The data in the fact table in the model generally increases over time, such as new orders grow over time in the order table. Then the order date can be the time partition column to partition orders into daily incremental batches. Setting the time partition column occurs after saving the model design.
- - Full load: When the model does not have a time partition column, then data in fact table is loaded fully every time. If you need to load the latest week of data in the order table, all data will be reloaded because the model does not have a time partition column. You can find out more in the [Full Load](../modeling/load_data/full_build.md) section.
- - Incremental load: When the model has been built and put into business analysis, and the model has a time partition column, you can still incrementally load new data while serving queries. For example, new data in the order table can be loaded incrementally daily. Incremental loading eliminates the need to reload pre-calculated data, this increases productivity and saves resources. You can find out more by looking at the [Load by Date/Time](../modeling/load_data/by_date.md) section.
+ - Full load: When the model does not have a time partition column, then data in fact table is loaded fully every time. If you need to load the latest week of data in the order table, all data will be reloaded because the model does not have a time partition column. You can find out more in the [Full Load](../model/load_data/full_build.md) section.
+ - Incremental load: When the model has been built and put into business analysis, and the model has a time partition column, you can still incrementally load new data while serving queries. For example, new data in the order table can be loaded incrementally daily. Incremental loading eliminates the need to reload pre-calculated data, this increases productivity and saves resources. You can find out more by looking at the [Load by Date/Time](../model/load_data/by_date.md) section.
2. **How to Load Data**
diff --git a/website/docs/restapi/error_code.md b/website/docs/restapi/error_code.md
index f825866ba8..216938d4ab 100644
--- a/website/docs/restapi/error_code.md
+++ b/website/docs/restapi/error_code.md
@@ -111,7 +111,7 @@ If an error occurs when calling the Kylin API or using the built-in tools, you c
| KE-010043208 | The entered parameter value is invalid. The parameter value must be a non-negative <br />integer. Please check and try again. |
| KE-010043209 | The entered parameter value is invalid. Only support specific values at the moment. Please <br />check and try again. |
| KE-010043210 | The parameter can't be empty. Please enter the time partition column format. |
-| KE-010043211 | The type of the time partition column is invalid. Please enter the supported format, refer <br />to the [user manual](../modeling/intro.md). |
+| KE-010043211 | The type of the time partition column is invalid. Please enter the supported format, refer <br />to the [user manual](../model/intro.md). |
| KE-010043212 | The parameter can't be empty. Please enter layout(s) id. |
| KE-010043213 | Can't find layout. Please check and try again. |
| KE-010043214 | Can't refresh the value, the time units are only supported in d (days), h (hours), <br />or m (minutes). Please check and try again. |
diff --git a/website/docs/restapi/model_api/intro.md b/website/docs/restapi/model_api/intro.md
index 66a16099a7..d5ffff5bfa 100644
--- a/website/docs/restapi/model_api/intro.md
+++ b/website/docs/restapi/model_api/intro.md
@@ -89,7 +89,7 @@ Kylin provides REST APIs on checking model information, index build and model ma
- `primary_key` - `required` `string[]`, primary key
- `simplified_non_equi_join_conditions` - `optional` `JSON Object`, non-equivalent join conditions
- (note1: The support of this settings should have 'Support History Table' enabled in advance. Seeing [Slowly Changing Dimension](../../modeling/model_design/slowly_changing_dimension.md)
+ (note1: The support of this settings should have 'Support History Table' enabled in advance. Seeing [Slowly Changing Dimension](../../model/model_design/scd2)
(note2: Join relationship >= and < must be used in pairs, and same column must be joint in both conditions)
@@ -102,7 +102,7 @@ Kylin provides REST APIs on checking model information, index build and model ma
- `join_relation_type` - `optional` `string`, join type: MANY_TO_ONE,MANY_TO_MANY, default: MANY_TO_ONE
- `partition_desc` - `optional` `JSON Object`, partition columns info
- `partition_date_column` - `required` `string`, partition date column, format: TABLE.COLUMN
- - `partition_date_format` - `required` `string`, partition date column format, including: yyyy-MM-dd, yyyyMMdd... Supported date format please check "[Model Design Basics](../../modeling/intro.md)"
+ - `partition_date_format` - `required` `string`, partition date column format, including: yyyy-MM-dd, yyyyMMdd... Supported date format please check "[Model Design Basics](../../model/intro.md)"
- `partition_type` - `optional` `string`, partition type, including: APPEND, default: APPEND
- `owner` - `optional` `string`, the owner of model, default current user
- `description` - `optional` `string`, model description
@@ -367,7 +367,7 @@ Kylin provides REST APIs on checking model information, index build and model ma
> [!NOTE]
>
- > For parameters not listed in this page, you may go to the [Model Concepts and Operations](../../modeling/model_management.md) for details.
+ > For parameters not listed in this page, you may go to the [Model Concepts and Operations](../../model/model_management.md) for details.
```json
{
@@ -511,7 +511,7 @@ Kylin provides REST APIs on checking model information, index build and model ma
- `hierarchy_dims`: `JSON`, dimension hierarchy
- `mandatory_dims`: `JSON`, required hierarchy
- `joint_dims`: `JSON`, joint dimensions
- - `computed_columns`: `JSON`, [Computed Column](../../modeling/model_design/computed_column.md)
+ - `computed_columns`: `JSON`, [Computed Column](../../model/model_design/computed_column.md)
- `tableIdentity`: `String`, table ID
- `tableAlias`: `String`, table alias
- `columnName`: `String`, column name
diff --git a/website/sidebars.ts b/website/sidebars.ts
index 7f54bcfee8..bc261d1306 100644
--- a/website/sidebars.ts
+++ b/website/sidebars.ts
@@ -68,10 +68,6 @@ const sidebars = {
type: 'doc',
id: 'deployment/on-premises/rdbms_metastore/usepg_as_metadb'
},
- {
- type: 'doc',
- id: 'deployment/on-premises/rdbms_metastore/query_history_fields'
- },
]
},
{
@@ -215,193 +211,158 @@ const sidebars = {
}
],
},
+ //////////////////////////////////////// Model category \\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\
{
type: 'category',
label: 'Model',
link: {
type: 'doc',
- id: 'modeling/intro'
+ id: 'model/intro'
},
items: [
- {
- type: 'doc',
- id: 'modeling/manual_modeling'
- },
{
type: 'category',
- label: 'Auto Modeling',
+ label: 'Manual Modeling',
link: {
type: 'doc',
- id: 'modeling/auto_modeling/intro'
+ id: 'model/manual/modeling'
},
items: [
{
type: 'doc',
- id: 'modeling/auto_modeling/data_modeling_by_SQL'
+ id: 'model/manual/aggregation_group'
},
{
type: 'doc',
- id: 'modeling/auto_modeling/basic_concept_actions'
+ id: 'model/manual/table_index'
},
{
type: 'doc',
- id: 'modeling/auto_modeling/rule_setting'
+ id: 'model/manual/computed_column'
},
- {
- type: 'doc',
- id: 'modeling/auto_modeling/scalar_subquery'
- },
- ],
+ ]
},
{
type: 'category',
- label: 'Auto Index Optimization',
+ label: 'Recommendation',
link: {
type: 'doc',
- id: 'modeling/auto_modeling/optimize_index/intro'
+ id: 'model/rec/intro'
},
items: [
{
type: 'doc',
- id: 'modeling/auto_modeling/optimize_index/index_optimization'
+ id: 'model/rec/sql_modeling'
},
{
type: 'doc',
- id: 'modeling/auto_modeling/optimize_index/review_index'
+ id: 'model/rec/optimize_by_qh'
+ },
+ {
+ type: 'doc',
+ id: 'model/rec/index_optimization'
},
],
},
{
- type: 'doc',
- id: 'modeling/model_management'
+ type: 'category',
+ label: 'Management',
+ items: [
+ {
+ type: 'doc',
+ id: 'model/manage/model_info'
+ },
+ {
+ type: 'doc',
+ id: 'model/manage/model_operation'
+ },
+ {
+ type: 'doc',
+ id: 'model/manage/import_export'
+ },
+ {
+ type: 'doc',
+ id: 'model/manage/data_loading'
+ },
+ {
+ type: 'doc',
+ id: 'model/manage/segment'
+ },
+ /*{
+ type: 'doc',
+ id: 'model/manage/configuration'
+ }*/
+ ],
},
{
type: 'doc',
- id: 'modeling/snapshot_management'
+ id: 'model/snapshot/snapshot'
},
{
type: 'category',
- label: 'Advanced Mode Design',
- link: {
- type: 'doc',
- id: 'modeling/model_design/intro'
- },
+ label: 'Advanced Features',
items: [
{
type: 'category',
label: 'Measures',
link: {
type: 'doc',
- id: 'modeling/model_design/measure_design/intro'
+ id: 'model/features/measures/intro'
},
items: [
{
type: 'doc',
- id: 'modeling/model_design/measure_design/topn'
+ id: 'model/features/measures/topn'
},
{
type: 'doc',
- id: 'modeling/model_design/measure_design/count_distinct_bitmap'
+ id: 'model/features/measures/count_distinct_bitmap'
},
{
type: 'doc',
- id: 'modeling/model_design/measure_design/count_distinct_hllc'
+ id: 'model/features/measures/count_distinct_hllc'
},
{
type: 'doc',
- id: 'modeling/model_design/measure_design/percentile_approx'
+ id: 'model/features/measures/percentile_approx'
},
{
type: 'doc',
- id: 'modeling/model_design/measure_design/corr'
+ id: 'model/features/measures/corr'
},
{
type: 'doc',
- id: 'modeling/model_design/measure_design/collect_set'
+ id: 'model/features/measures/collect_set'
},
],
},
{
type: 'doc',
- id: 'modeling/model_design/computed_column'
- },
- {
- type: 'doc',
- id: 'modeling/model_design/slowly_changing_dimension'
- },
- {
- type: 'doc',
- id: 'modeling/model_design/aggregation_group'
+ id: 'model/features/scd2'
},
+ // {
+ // type: 'doc',
+ // id: 'model/features/excluded_column'
+ // },
{
type: 'doc',
- id: 'modeling/model_design/table_index'
- },
- {
- type: 'category',
- label: 'Model Advanced Settings',
- link: {
- type: 'doc',
- id: 'modeling/model_design/advance_guide/intro'
- },
- items: [
- {
- type: 'doc',
- id: 'modeling/model_design/advance_guide/model_metadata_managment'
- },
- {
- type: 'doc',
- id: 'modeling/model_design/advance_guide/multilevel_partitioning'
- },
- {
- type: 'doc',
- id: 'modeling/model_design/advance_guide/fast_bitmap'
- },
- {
- type: 'doc',
- id: 'modeling/model_design/advance_guide/integer_encoding'
- },
- ],
+ id: 'model/features/runtime_join'
},
{
type: 'doc',
- id: 'modeling/model_design/precompute_join_relations'
+ id: 'model/features/multi_partitioning'
},
- ],
- },
- {
- type: 'category',
- label: 'Data Loading',
- link: {
- type: 'doc',
- id: 'modeling/load_data/intro'
- },
- items: [
{
type: 'doc',
- id: 'modeling/load_data/full_build'
+ id: 'model/features/scalar_subquery'
},
{
type: 'doc',
- id: 'modeling/load_data/by_date'
+ id: 'model/features/fast_bitmap'
},
{
type: 'doc',
- id: 'modeling/load_data/build_index'
- },
- {
- type: 'category',
- label: 'Segment Operation and Settings',
- link: {
- type: 'doc',
- id: 'modeling/load_data/segment_operation_settings/intro'
- },
- items: [
- {
- type: 'doc',
- id: 'modeling/load_data/segment_operation_settings/segment_merge'
- },
- ],
+ id: 'model/features/integer_encoding'
},
],
},
@@ -410,26 +371,26 @@ const sidebars = {
label: 'Streaming',
link: {
type: 'doc',
- id: 'modeling/real-time/intro'
+ id: 'model/streaming/intro'
},
items: [
{
type: 'doc',
- id: 'modeling/real-time/prerequisite'
+ id: 'model/streaming/prerequisite'
},
{
type: 'doc',
- id: 'modeling/real-time/real-time'
+ id: 'model/streaming/real-time'
},
{
type: 'doc',
- id: 'modeling/real-time/custom-parser'
+ id: 'model/streaming/custom-parser'
},
],
},
],
},
-
+ //////////////////////////////////////// Query Category \\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\
{
type: 'category',
label: 'Query',
@@ -518,9 +479,20 @@ const sidebars = {
id: 'query/push_down'
},
{
- type: 'doc',
- id: 'query/query_history'
+ type: 'category',
+ label: 'Query History',
+ items: [
+ {
+ type: 'doc',
+ id: 'query/history/query_history'
+ },
+ {
+ type: 'doc',
+ id: 'query/history/query_history_fields'
+ },
+ ],
},
+
{
type: 'doc',
id: 'query/async_query'