You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by ni...@apache.org on 2019/12/17 08:01:48 UTC
[kylin] branch master updated (52788ae -> cedc430)
This is an automated email from the ASF dual-hosted git repository.
nic pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git.
from 52788ae KYLIN-4226: Skip current unavailable tables when updating hbase coprocessor
new 2b830a3 KYLIN-4211 PartitionDesc support custom year、month、day partitions name
new 28e6a33 KYLIN-4211 Add unit test case for CustomYearMonthDayFieldPartitionConditionBuilder
new cedc430 KYLIN-4211 Front end for the feature of setting date partition in different columns
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../apache/kylin/metadata/model/PartitionDesc.java | 63 +++++++++++++++++++---
.../kylin/metadata/model/PartitionDescTest.java | 61 +++++++++++++++++++++
.../app/js/controllers/modelConditionsSettings.js | 25 ++++++++-
.../modelDesigner/conditions_settings.html | 63 ++++++++++++++++++++--
4 files changed, 200 insertions(+), 12 deletions(-)
create mode 100644 core-metadata/src/test/java/org/apache/kylin/metadata/model/PartitionDescTest.java
[kylin] 03/03: KYLIN-4211 Front end for the feature of setting date
partition in different columns
Posted by ni...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
nic pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit cedc430f7125b89bc17ca9d7b92f6581f33b4e48
Author: nichunen <ni...@apache.org>
AuthorDate: Tue Dec 17 16:00:38 2019 +0800
KYLIN-4211 Front end for the feature of setting date partition in different columns
---
.../app/js/controllers/modelConditionsSettings.js | 25 ++++++++-
.../modelDesigner/conditions_settings.html | 63 ++++++++++++++++++++--
2 files changed, 82 insertions(+), 6 deletions(-)
diff --git a/webapp/app/js/controllers/modelConditionsSettings.js b/webapp/app/js/controllers/modelConditionsSettings.js
index 500db40..42ffa98 100644
--- a/webapp/app/js/controllers/modelConditionsSettings.js
+++ b/webapp/app/js/controllers/modelConditionsSettings.js
@@ -30,6 +30,7 @@ KylinApp.controller('ModelConditionsSettingsCtrl', function ($scope, $modal,Meta
// firstValue: For fixing ng-chosen cannot watch first value change.
$scope.partition_date = { type: '', format: '', firstValue: '' };
$scope.partition_time = { type: '', format: '', firstValue: '' };
+ $scope.partition_ymd_columns = { year_column: '', month_column: '', day_column: ''};
$scope.initSetting = function (){
$scope.selectedTables={fact:VdmUtil.getNameSpaceAliasName($scope.modelsManager.selectedModel.partition_desc.partition_date_column)}
@@ -160,8 +161,19 @@ KylinApp.controller('ModelConditionsSettingsCtrl', function ($scope, $modal,Meta
return;
}
};
+
+ $scope.partitionYMDColumnChange = function (type, value) {
+ if (type === 'Y') {
+ $scope.modelsManager.selectedModel.partition_desc.partition_date_column = value + ',' + $scope.partition_ymd_columns.month_column + ',' + $scope.partition_ymd_columns.day_column;
+ } else if (type === 'M') {
+ $scope.modelsManager.selectedModel.partition_desc.partition_date_column = $scope.partition_ymd_columns.year_column + ',' + value + ',' + $scope.partition_ymd_columns.day_column;
+ } else if (type === 'D') {
+ $scope.modelsManager.selectedModel.partition_desc.partition_date_column = $scope.partition_ymd_columns.year_column + ',' + $scope.partition_ymd_columns.month_column + ',' + value;
+ }
+ };
- $scope.partitionColumn ={
+ $scope.partitionColumn = {
+ "hasSeparateDateColumns": false,
"hasSeparateTimeColumn" : false
}
@@ -186,9 +198,18 @@ KylinApp.controller('ModelConditionsSettingsCtrl', function ($scope, $modal,Meta
if($scope.modelsManager.selectedModel.partition_desc.partition_time_column){
$scope.partitionColumn.hasSeparateTimeColumn = true;
}
- $scope.toggleHasSeparateColumn = function(){
+ $scope.toggleHasSeparateTimeColumn = function(){
if($scope.partitionColumn.hasSeparateTimeColumn == false){
$scope.modelsManager.selectedModel.partition_desc.partition_time_column = null;
}
}
+ $scope.toggleHasSeparateDateColumns = function(){
+ $scope.modelsManager.selectedModel.partition_desc.partition_date_column = null;
+ if($scope.partitionColumn.hasSeparateDateColumns == false){
+ $scope.partition_ymd_columns = { year_column: '', month_column: '', day_column: ''};
+ } else {
+ $scope.partition_date.type = 'yyyy-MM-dd';
+ $scope.modelsManager.selectedModel.partition_desc.partition_condition_builder = 'org.apache.kylin.metadata.model.PartitionDesc$CustomYearMonthDayFieldPartitionConditionBuilder';
+ }
+ }
});
diff --git a/webapp/app/partials/modelDesigner/conditions_settings.html b/webapp/app/partials/modelDesigner/conditions_settings.html
index becb327..69610e4 100644
--- a/webapp/app/partials/modelDesigner/conditions_settings.html
+++ b/webapp/app/partials/modelDesigner/conditions_settings.html
@@ -22,8 +22,61 @@
<h3>Partition</h3>
<div class="box-body">
+ <!--Date Format-->
+ <div class="form-group middle-popover">
+ <div class="row">
+ <label class="control-label col-xs-12 col-sm-3 no-padding-right font-color-default"><b>Has separate columns for "year", "month" and "day" ?</b> <i kylinpopover placement="right" title="Separate Date Columns" template="separateDateColumnsTip.html" class="fa fa-info-circle"></i></label>
+ <div class="col-xs-12 col-sm-6">
+ <toggle-switch on-label="Yes" off-label="No" ng-disabled="state.mode!=='edit'" ng-click="toggleHasSeparateDateColumns()" ng-model="partitionColumn.hasSeparateDateColumns"></toggle-switch>
+ </div>
+ </div>
+ </div>
+
+ <!--Partition Columns-->
+ <div class="form-group" ng-if="partitionColumn.hasSeparateDateColumns">
+ <div class="row middle-popover">
+ <label class="control-label col-xs-12 col-sm-3 no-padding-right font-color-default"><b>Partition Date Columns</b> <i kylinpopover placement="right" title="Partition Date Columns" class="fa fa-info-circle"></i></label>
+ <div class="col-xs-12 col-sm-6" ng-if="state.mode=='edit'">
+ <select width="'49%'" chosen data-placeholder="e.g. DEFAULT.TABLE_FACT"
+ ng-model="selectedTables.fact"
+ ng-change="tableChange(selectedTables.fact)"
+ data-placement=""
+ ng-options="alias as alias for alias in availableFactTables" >
+ <option value="">--Select Partition Table--</option>
+ </select>
+
+ <select width="'49%'" chosen data-placeholder="e.g. DEFAULT.TABLE_FACT.YEAR"
+ ng-model="partition_ymd_columns.year_column"
+ ng-change="partitionYMDColumnChange('Y', partition_ymd_columns.year_column)"
+ data-placement=""
+ ng-options="selectedTables.fact+'.'+columns.name as columns.name for columns in getPartitonColumns(selectedTables.fact)" >
+ <option value="">--Select Partition Column for Year--</option>
+ </select>
+ <select width="'49%'" chosen data-placeholder="e.g. DEFAULT.TABLE_FACT.MONTH"
+ ng-model="partition_ymd_columns.month_column"
+ ng-change="partitionYMDColumnChange('M', partition_ymd_columns.month_column)"
+ data-placement=""
+ ng-options="selectedTables.fact+'.'+columns.name as columns.name for columns in getPartitonColumns(selectedTables.fact)" >
+ <option value="">--Select Partition Column for Month--</option>
+ </select>
+ <select width="'49%'" chosen data-placeholder="e.g. DEFAULT.TABLE_FACT.DAY"
+ ng-model="partition_ymd_columns.day_column"
+ ng-change="partitionYMDColumnChange('D', partition_ymd_columns.day_column)"
+ data-placement=""
+ ng-options="selectedTables.fact+'.'+columns.name as columns.name for columns in getPartitonColumns(selectedTables.fact)" >
+ <option value="">--Select Partition Column for Day--</option>
+ </select>
+ </div>
+ <div class="col-xs-12 col-sm-6" ng-if="state.mode=='view'">
+ <span >
+ {{!!(modelsManager.selectedModel.partition_desc.partition_date_column)?modelsManager.selectedModel.partition_desc.partition_date_column: ''}}
+ </span>
+ </div>
+ </div>
+ </div>
+
<!--Partition Column-->
- <div class="form-group">
+ <div class="form-group" ng-if="!partitionColumn.hasSeparateDateColumns">
<div class="row middle-popover">
<label class="control-label col-xs-12 col-sm-3 no-padding-right font-color-default"><b>Partition Date Column</b> <i kylinpopover placement="right" title="Partition Date Column" template="partitionTip.html" class="fa fa-info-circle"></i></label>
<div class="col-xs-12 col-sm-6" ng-if="state.mode=='edit'">
@@ -62,7 +115,7 @@
ng-if="state.mode=='edit'"
ng-model="partition_date.type"
ng-required="partition_date.type"
- ng-disabled="$parent.isFormatEdit.editable!==true"
+ ng-disabled="$parent.isFormatEdit.editable!==true || partitionColumn.hasSeparateDateColumns"
ng-class="{ 'col-5': partition_date.type === 'other' }"
ng-options="ddt as ddt for ddt in cubeConfig.partitionDateFormatOpt"
data-placement="">
@@ -85,7 +138,7 @@
<div class="row">
<label class="control-label col-xs-12 col-sm-3 no-padding-right font-color-default"><b>Has a separate "time of the day" column ?</b> <i kylinpopover placement="right" title="Separate Time Column" template="separateTimeColumnTip.html" class="fa fa-info-circle"></i></label>
<div class="col-xs-12 col-sm-6">
- <toggle-switch on-label="Yes" off-label="No" ng-disabled="state.mode!=='edit'" ng-click="toggleHasSeparateColumn()" ng-model="partitionColumn.hasSeparateTimeColumn"></toggle-switch>
+ <toggle-switch on-label="Yes" off-label="No" ng-disabled="state.mode!=='edit'" ng-click="toggleHasSeparateTimeColumn()" ng-model="partitionColumn.hasSeparateTimeColumn"></toggle-switch>
</div>
</div>
</div>
@@ -198,4 +251,6 @@
<script type="text/ng-template" id="separateTimeColumnTip.html">
<p>For cases where fact table saves date and hour at two columns (KYLIN-1427)</p>
</script>
-
+<script type="text/ng-template" id="separateDateColumnsTip.html">
+ <p>For cases where fact table saves date (year, month and day) in separate columns (KYLIN-4211)</p>
+</script>
[kylin] 01/03: KYLIN-4211 PartitionDesc support custom year、month、day partitions name
Posted by ni...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
nic pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 2b830a3e6d6e17d488af307d030e0d73d094aa74
Author: bigxiaochu <cr...@163.com>
AuthorDate: Tue Oct 29 11:59:26 2019 +0800
KYLIN-4211 PartitionDesc support custom year、month、day partitions name
---
.../apache/kylin/metadata/model/PartitionDesc.java | 63 +++++++++++++++++++---
1 file changed, 57 insertions(+), 6 deletions(-)
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java
index f93996e..a4c4cc7 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java
@@ -71,13 +71,19 @@ public class PartitionDesc implements Serializable {
if (StringUtils.isEmpty(partitionDateColumn))
return;
- partitionDateColumnRef = model.findColumn(partitionDateColumn);
- partitionDateColumn = partitionDateColumnRef.getIdentity();
- if (StringUtils.isBlank(partitionTimeColumn) == false) {
- partitionTimeColumnRef = model.findColumn(partitionTimeColumn);
- partitionTimeColumn = partitionTimeColumnRef.getIdentity();
- }
+ //Support CustomYearMonthDayFieldPartitionConditionBuilder, partitionDateColumn split by ","
partitionConditionBuilder = (IPartitionConditionBuilder) ClassUtil.newInstance(partitionConditionBuilderClz);
+ if (partitionConditionBuilder instanceof CustomYearMonthDayFieldPartitionConditionBuilder) {
+ ((CustomYearMonthDayFieldPartitionConditionBuilder)partitionConditionBuilder).init(this, model);
+ } else {
+ partitionDateColumnRef = model.findColumn(partitionDateColumn);
+ partitionDateColumn = partitionDateColumnRef.getIdentity();
+ if (StringUtils.isBlank(partitionTimeColumn) == false) {
+ partitionTimeColumnRef = model.findColumn(partitionTimeColumn);
+ partitionTimeColumn = partitionTimeColumnRef.getIdentity();
+ }
+ }
+
}
public boolean partitionColumnIsYmdInt() {
@@ -340,6 +346,51 @@ public class PartitionDesc implements Serializable {
}
}
+ /**
+ * Another implementation of IPartitionConditionBuilder, for the fact tables which have three custom partition columns like "Y", "M", "D" means "YEAR", "MONTH", and "DAY";
+ * This class will conicat the three columns into yyyy-MM-dd format for query hive;
+ * implements Serializable for spark build
+ */
+ public static class CustomYearMonthDayFieldPartitionConditionBuilder implements IPartitionConditionBuilder, Serializable {
+ private String yearPartitionDateColumn;
+ private String monthPartitionDateColumn;
+ private String dayPartitionDateColumn;
+ @Override
+ public String buildDateRangeCondition(PartitionDesc partDesc, ISegment seg, SegmentRange segRange, Function<TblColRef, String> func) {
+ long startInclusive = (Long) segRange.start.v;
+ long endExclusive = (Long) segRange.end.v;
+
+ TblColRef partitionColumn = partDesc.getPartitionDateColumnRef();
+ if (partitionColumn != null) {
+ partitionColumn.setQuotedFunc(func);
+ }
+ String concatField = String.format(Locale.ROOT, "CONCAT(%s,'-',%s,'-',%s)", yearPartitionDateColumn,
+ monthPartitionDateColumn, dayPartitionDateColumn);
+ StringBuilder builder = new StringBuilder();
+
+ if (startInclusive > 0) {
+ builder.append(concatField + " >= '" + DateFormat.formatToDateStr(startInclusive) + "' ");
+ builder.append("AND ");
+ }
+ builder.append(concatField + " < '" + DateFormat.formatToDateStr(endExclusive) + "'");
+
+ return builder.toString();
+ }
+
+ public void init(PartitionDesc partitionDesc, DataModelDesc model) {
+ String[] yearMonthDayColumns = partitionDesc.getPartitionDateColumn().split(",");
+ if (yearMonthDayColumns.length != 3) {
+ throw new IllegalArgumentException(partitionDesc.getPartitionDateColumn() + " is not year, month, and day columns");
+ }
+ TblColRef yearRef = model.findColumn(yearMonthDayColumns[0]);
+ yearPartitionDateColumn = yearRef.getIdentity();
+ monthPartitionDateColumn = model.findColumn(yearMonthDayColumns[1]).getIdentity();
+ dayPartitionDateColumn = model.findColumn(yearMonthDayColumns[2]).getIdentity();
+ //for partition desc isPartitioned() true
+ partitionDesc.setPartitionDateColumnRef(yearRef);
+ }
+ }
+
public static PartitionDesc getCopyOf(PartitionDesc orig) {
PartitionDesc ret = new PartitionDesc();
ret.partitionDateColumn = orig.partitionDateColumn;
[kylin] 02/03: KYLIN-4211 Add unit test case for
CustomYearMonthDayFieldPartitionConditionBuilder
Posted by ni...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
nic pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 28e6a333960675721e39485319aa0cf516bbed98
Author: nichunen <ni...@apache.org>
AuthorDate: Tue Dec 17 13:09:06 2019 +0800
KYLIN-4211 Add unit test case for CustomYearMonthDayFieldPartitionConditionBuilder
---
.../kylin/metadata/model/PartitionDescTest.java | 61 ++++++++++++++++++++++
1 file changed, 61 insertions(+)
diff --git a/core-metadata/src/test/java/org/apache/kylin/metadata/model/PartitionDescTest.java b/core-metadata/src/test/java/org/apache/kylin/metadata/model/PartitionDescTest.java
new file mode 100644
index 0000000..e537b07
--- /dev/null
+++ b/core-metadata/src/test/java/org/apache/kylin/metadata/model/PartitionDescTest.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.metadata.model;
+
+import java.io.IOException;
+
+import org.apache.kylin.common.util.LocalFileMetadataTestCase;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+public class PartitionDescTest extends LocalFileMetadataTestCase {
+
+ @Before
+ public void setUp() throws Exception {
+ this.createTestMetadata();
+ }
+
+ @After
+ public void after() throws Exception {
+ this.cleanupTestMetadata();
+ }
+
+ @Test
+ public void testCustomYearMonthDayFieldPartitionConditionBuilder() throws IOException {
+ DataModelManager dataModelManager = DataModelManager.getInstance(getTestConfig());
+ DataModelDesc model = dataModelManager.getDataModelDesc("test_kylin_inner_join_model_desc");
+ PartitionDesc yearMonthDayPartitionDesc = new PartitionDesc();
+ yearMonthDayPartitionDesc.setPartitionDateColumn(
+ "TEST_KYLIN_FACT.LSTG_SITE_ID,TEST_KYLIN_FACT.LEAF_CATEG_ID,TEST_KYLIN_FACT.SELLER_ID");
+ yearMonthDayPartitionDesc.setPartitionConditionBuilderClz(
+ "org.apache.kylin.metadata.model.PartitionDesc$CustomYearMonthDayFieldPartitionConditionBuilder");
+ model.setPartitionDesc(yearMonthDayPartitionDesc);
+ dataModelManager.updateDataModelDesc(model);
+ PartitionDesc.CustomYearMonthDayFieldPartitionConditionBuilder builder = (PartitionDesc.CustomYearMonthDayFieldPartitionConditionBuilder) model
+ .getPartitionDesc().getPartitionConditionBuilder();
+ String dateRangeCondition = builder.buildDateRangeCondition(model.getPartitionDesc(), null,
+ new SegmentRange(0l, 86400000l), null);
+ Assert.assertEquals(
+ "CONCAT(TEST_KYLIN_FACT.LSTG_SITE_ID,'-',TEST_KYLIN_FACT.LEAF_CATEG_ID,'-',TEST_KYLIN_FACT.SELLER_ID) < '1970-01-02'",
+ dateRangeCondition);
+ }
+
+}