You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2018/07/19 07:18:04 UTC
[kylin] branch master updated (85c3895 -> 7f2aa5a)
This is an automated email from the ASF dual-hosted git repository.
shaofengshi pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git.
from 85c3895 KYLIN-3255 cannot save cube and unit test
new 4b5fe6d KYLIN-3186 add partition_time other feature, abstract the 'initialPartitionSetting' method and enable the partition format editable of all data type
new 7f2aa5a KYLIN-3186 expand partition-date-column format
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../org/apache/kylin/common/util/DateFormat.java | 30 +++++-
.../apache/kylin/metadata/model/PartitionDesc.java | 8 +-
.../app/js/controllers/modelConditionsSettings.js | 113 ++++++++++++++++++---
webapp/app/js/model/cubeConfig.js | 12 ++-
webapp/app/less/app.less | 14 +++
.../modelDesigner/conditions_settings.html | 48 ++++++---
6 files changed, 186 insertions(+), 39 deletions(-)
[kylin] 01/02: KYLIN-3186 add partition_time other feature,
abstract the 'initialPartitionSetting' method and enable the
partition format editable of all data type
Posted by sh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
shaofengshi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 4b5fe6de6278f54751ed9a58c336b133895b05c9
Author: Emiya0306 <wo...@qq.com>
AuthorDate: Mon Jul 16 17:10:09 2018 +0800
KYLIN-3186 add partition_time other feature, abstract the 'initialPartitionSetting' method and enable the partition format editable of all data type
- enable the partition format editable of all data type
- optimize stylesheet for responsive style
- add partition_time other feature and abstract the 'initialPartitionSetting' method but not enabled
---
.../app/js/controllers/modelConditionsSettings.js | 113 ++++++++++++++++++---
webapp/app/js/model/cubeConfig.js | 12 ++-
webapp/app/less/app.less | 14 +++
.../modelDesigner/conditions_settings.html | 48 ++++++---
4 files changed, 155 insertions(+), 32 deletions(-)
diff --git a/webapp/app/js/controllers/modelConditionsSettings.js b/webapp/app/js/controllers/modelConditionsSettings.js
index 82abe0b..500db40 100644
--- a/webapp/app/js/controllers/modelConditionsSettings.js
+++ b/webapp/app/js/controllers/modelConditionsSettings.js
@@ -25,6 +25,12 @@
KylinApp.controller('ModelConditionsSettingsCtrl', function ($scope, $modal,MetaModel,modelsManager,VdmUtil) {
$scope.modelsManager = modelsManager;
$scope.availableFactTables = [];
+ // partition date temporary object.
+ // Because ng-chosen cannot watch string value, partition date should be object.
+ // firstValue: For fixing ng-chosen cannot watch first value change.
+ $scope.partition_date = { type: '', format: '', firstValue: '' };
+ $scope.partition_time = { type: '', format: '', firstValue: '' };
+
$scope.initSetting = function (){
$scope.selectedTables={fact:VdmUtil.getNameSpaceAliasName($scope.modelsManager.selectedModel.partition_desc.partition_date_column)}
$scope.selectedTablesForPartitionTime={fact:VdmUtil.getNameSpaceAliasName($scope.modelsManager.selectedModel.partition_desc.partition_time_column)}
@@ -35,31 +41,94 @@ KylinApp.controller('ModelConditionsSettingsCtrl', function ($scope, $modal,Meta
$scope.availableFactTables.push(joinTable[j].alias);
}
}
+
+ $scope.initialPartitionSetting('Date');
+ $scope.initialPartitionSetting('Time');
}
$scope.isFormatEdit = {editable:false};
+
+ /**
+ * initial date or time partition select
+ *
+ * @param {String: 'Date' | 'Time'} partitionFieldName
+ * @desc cubeConfigName: 'partitionDateFormatOpt' or 'partitionTimeFormatOpt'
+ * modelFormatKey: 'partition_date_format' or 'partition_time_format'
+ * scopeName: 'partition_date' or 'partition_time'
+ */
+ $scope.initialPartitionSetting = function(partitionFieldName) {
+ var cubeConfigName = 'partition' + partitionFieldName + 'FormatOpt';
+
+ var lowerCaseName = partitionFieldName.toLowerCase(),
+ modelFormatKey = 'partition_' + lowerCaseName + '_format',
+ scopeName = 'partition_' + lowerCaseName,
+ scopePartitionTypeWatchName = 'partition_' + lowerCaseName + '.type',
+ scopePartitionFormatWatchName = 'partition_' + lowerCaseName + '.format';
+
+ var partitionFormatOpt = $scope.cubeConfig[cubeConfigName];
+ var partition_format = $scope.modelsManager.selectedModel.partition_desc[modelFormatKey];
+
+ if(partitionFormatOpt.indexOf(partition_format) === -1) {
+ $scope[scopeName].type = 'other';
+ $scope[scopeName].format = partition_format;
+ $scope[scopeName].firstValue = partition_format;
+ } else {
+ $scope[scopeName].type = partition_format;
+ $scope[scopeName].format = '';
+ $scope[scopeName].firstValue = partition_format;
+ }
+
+ // Add form change watcher. SetTimeout can escape the first render loop.
+ setTimeout(function() {
+ $scope.$watch(scopePartitionTypeWatchName, function (newValue, oldValue) {
+ // Ng-chosen will change the value of all selects on DOM when you select each first.
+ // So for fixing this bug, we should compare the newValue and oldValue.
+ // firstValue: For fixing ng-chosen cannot watch first value change.
+ if(newValue !== oldValue || ($scope[scopeName].firstValue && $scope[scopeName].firstValue !== newValue && newValue !== 'other')) {
+ if(newValue !== 'other') {
+ $scope.modelsManager.selectedModel.partition_desc[modelFormatKey] = $scope[scopeName].format = newValue;
+ } else {
+ $scope[scopeName].format = '';
+ }
+ $scope[scopeName].firstValue = '';
+ }
+ });
+
+ $scope.$watch(scopePartitionFormatWatchName, function (newValue, oldValue) {
+ if(newValue !== oldValue) {
+ $scope.modelsManager.selectedModel.partition_desc[modelFormatKey] = newValue;
+ }
+ });
+ });
+ };
+
var judgeFormatEditable = function(dateColumn){
if(dateColumn == null){
$scope.isFormatEdit.editable = false;
return;
}
- var column = _.filter($scope.getColumnsByAlias(VdmUtil.getNameSpaceAliasName(dateColumn)),function(_column){
- var columnName=VdmUtil.getNameSpaceAliasName(dateColumn)+"."+_column.name;
- if(dateColumn == columnName){
- return _column;
- }
- });
- var data_type = column[0].datatype;
- if(data_type ==="bigint" ||data_type ==="int" ||data_type ==="integer"){
- $scope.isFormatEdit.editable = false;
- $scope.modelsManager.selectedModel.partition_desc.partition_date_format='yyyyMMdd';
- $scope.partitionColumn.hasSeparateTimeColumn=false;
- $scope.modelsManager.selectedModel.partition_desc.partition_time_column=null;
- $scope.modelsManager.selectedModel.partition_desc.partition_time_format=null;
+ /**
+ * enable the partition format editable of all data type
+ * Edit date: 2018/07/12
+ * Author: Roger
+ */
+ // var column = _.filter($scope.getColumnsByAlias(VdmUtil.getNameSpaceAliasName(dateColumn)),function(_column){
+ // var columnName=VdmUtil.getNameSpaceAliasName(dateColumn)+"."+_column.name;
+ // if(dateColumn == columnName){
+ // return _column;
+ // }
+ // });
+ // var data_type = column[0].datatype;
+ // if(data_type ==="bigint" ||data_type ==="int" ||data_type ==="integer"){
+ // $scope.isFormatEdit.editable = false;
+ // $scope.modelsManager.selectedModel.partition_desc.partition_date_format='yyyyMMdd';
+ // $scope.partitionColumn.hasSeparateTimeColumn=false;
+ // $scope.modelsManager.selectedModel.partition_desc.partition_time_column=null;
+ // $scope.modelsManager.selectedModel.partition_desc.partition_time_format=null;
- return;
- }
+ // return;
+ // }
$scope.isFormatEdit.editable = true;
return;
@@ -96,6 +165,20 @@ KylinApp.controller('ModelConditionsSettingsCtrl', function ($scope, $modal,Meta
"hasSeparateTimeColumn" : false
}
+ $scope.addFormValueWatcher = function() {
+ $scope.$watch('partition_date.type', function (newValue) {
+ if(newValue !== 'other') {
+ $scope.modelsManager.selectedModel.partition_desc.partition_date_format = $scope.partition_date.format = newValue;
+ } else {
+ $scope.partition_date.format = '';
+ }
+ });
+
+ $scope.$watch('partition_date.format', function (newValue) {
+ $scope.modelsManager.selectedModel.partition_desc.partition_date_format = newValue;
+ });
+ };
+
if ($scope.state.mode=='edit'){
$scope.initSetting();
judgeFormatEditable($scope.modelsManager.selectedModel.partition_desc.partition_date_column);
diff --git a/webapp/app/js/model/cubeConfig.js b/webapp/app/js/model/cubeConfig.js
index eacb915..a83d4c9 100644
--- a/webapp/app/js/model/cubeConfig.js
+++ b/webapp/app/js/model/cubeConfig.js
@@ -96,14 +96,22 @@ KylinApp.constant('cubeConfig', {
{name:"minute_start",type:"timestamp"}
],
partitionDateFormatOpt:[
+ 'yyyy-MM-dd HH:mm:ss',
+ 'yyyy-MM-dd HH:mm',
+ 'yyyy-MM-dd HH',
'yyyy-MM-dd',
+ 'yyyyMMddHHMMSS',
+ 'yyyyMMddHHMM',
+ 'yyyyMMddHH',
'yyyyMMdd',
- 'yyyy-MM-dd HH:mm:ss'
+ // 'timestamp',
+ // 'other'
],
partitionTimeFormatOpt:[
'HH:mm:ss',
'HH:mm',
- 'HH'
+ 'HH',
+ // 'other'
],
rowKeyShardOptions:[
true,false
diff --git a/webapp/app/less/app.less b/webapp/app/less/app.less
index 7aa7283..cf9748e 100644
--- a/webapp/app/less/app.less
+++ b/webapp/app/less/app.less
@@ -936,4 +936,18 @@ td.snapshot-usage .tooltip {
}
td.snapshot-usage .tooltip-inner {
max-width: 1024px;
+}
+
+div[ng-controller="ModelConditionsSettingsCtrl"] {
+ .format-input {
+ margin-top: 15px;
+ }
+ select.col-5 + .chosen-container {
+ width: 49% !important;
+ }
+ .format-input.col-5 {
+ display: inline-block;
+ width: 49%;
+ margin-top: 0;
+ }
}
\ No newline at end of file
diff --git a/webapp/app/partials/modelDesigner/conditions_settings.html b/webapp/app/partials/modelDesigner/conditions_settings.html
index cce5de2..10aee84 100644
--- a/webapp/app/partials/modelDesigner/conditions_settings.html
+++ b/webapp/app/partials/modelDesigner/conditions_settings.html
@@ -27,7 +27,7 @@
<div class="row middle-popover">
<label class="control-label col-xs-12 col-sm-3 no-padding-right font-color-default"><b>Partition Date Column</b> <i kylinpopover placement="right" title="Partition Date Column" template="partitionTip.html" class="fa fa-info-circle"></i></label>
<div class="col-xs-12 col-sm-6" ng-if="state.mode=='edit'">
- <select style="width: 49%" chosen data-placeholder="e.g. DEFAULT.TEST_KYLIN_FACT.CAL_DT"
+ <select width="'49%'" chosen data-placeholder="e.g. DEFAULT.TEST_KYLIN_FACT.CAL_DT"
ng-model="selectedTables.fact"
ng-change="tableChange(selectedTables.fact)"
data-placement=""
@@ -35,7 +35,7 @@
<option value="">--Select Partition Table--</option>
</select>
- <select style="width: 49%" chosen data-placeholder="e.g. DEFAULT.TEST_KYLIN_FACT.CAL_DT"
+ <select width="'49%'" chosen data-placeholder="e.g. DEFAULT.TEST_KYLIN_FACT.CAL_DT"
ng-model="modelsManager.selectedModel.partition_desc.partition_date_column"
ng-change="partitionChange(modelsManager.selectedModel.partition_desc.partition_date_column)"
data-placement=""
@@ -57,15 +57,24 @@
<div class="row">
<label class="control-label col-xs-12 col-sm-3 no-padding-right font-color-default"><b>Date Format</b></label>
<div class="col-xs-12 col-sm-6">
- <select style="width: 100%" chosen
- ng-required="modelsManager.selectedModel.partition_desc.partition_date_format"
- ng-model="modelsManager.selectedModel.partition_desc.partition_date_format"
+ <select chosen
+ width="'98.5%'"
ng-if="state.mode=='edit'"
+ ng-model="partition_date.type"
+ ng-required="partition_date.type"
ng-disabled="$parent.isFormatEdit.editable!==true"
- data-placement=""
- ng-options="ddt as ddt for ddt in cubeConfig.partitionDateFormatOpt">
+ ng-class="{ 'col-5': partition_date.type === 'other' }"
+ ng-options="ddt as ddt for ddt in cubeConfig.partitionDateFormatOpt"
+ data-placement="">
<option value="">--Select Date Format--</option>
</select>
+ <input
+ class="form-control format-input"
+ ng-if="partition_date.type === 'other'"
+ ng-model="partition_date.format"
+ ng-required="partition_date.format"
+ ng-class="{ 'col-5': partition_date.type === 'other' }"
+ placeholder="Please enter your custom date format."/>
<span ng-if="state.mode=='view'&&modelsManager.selectedModel.partition_desc.partition_date_column">{{(modelsManager.selectedModel.partition_desc.partition_date_format)}}</span>
</div>
</div>
@@ -85,14 +94,14 @@
<div class="row middle-popover">
<label class="control-label col-xs-12 col-sm-3 no-padding-right font-color-default align-right"><b>Partition Time Column </b> <i kylinpopover placement="right" ng-title="Partition Time Column" template="partitionTimeTip.html" class="fa fa-info-circle"></i> </label>
<div class="col-xs-12 col-sm-6">
- <select style="width: 49%" chosen
+ <select width="'49%'" chosen
ng-model="selectedTablesForPartitionTime.fact"
ng-if="state.mode=='edit'"
data-placement=""
ng-options="alias as alias for alias in availableFactTables" >
<option value="">--Select Partition Table--</option>
</select>
- <select style="width: 49%" chosen
+ <select width="'49%'" chosen
ng-model="modelsManager.selectedModel.partition_desc.partition_time_column"
ng-if="state.mode=='edit'"
data-placement=""
@@ -109,14 +118,23 @@
<div class="row">
<label class="control-label col-xs-12 col-sm-3 no-padding-right font-color-default"><b>Time Format</b></label>
<div class="col-xs-12 col-sm-6">
- <select style="width: 100%" chosen
- ng-required="modelsManager.selectedModel.partition_desc.partition_time_format"
- ng-model="modelsManager.selectedModel.partition_desc.partition_time_format"
- ng-if="state.mode=='edit'"
- data-placement=""
- ng-options="ddt as ddt for ddt in cubeConfig.partitionTimeFormatOpt">
+ <select chosen
+ width="'98.5%'"
+ ng-if="state.mode=='edit'"
+ ng-model="partition_time.type"
+ ng-required="partition_time.type"
+ ng-class="{ 'col-5': partition_time.type === 'other' }"
+ ng-options="ddt as ddt for ddt in cubeConfig.partitionTimeFormatOpt"
+ data-placement="">
<option value="">--Select Time Format--</option>
</select>
+ <input
+ class="form-control format-input"
+ ng-if="partition_time.type === 'other'"
+ ng-model="partition_time.format"
+ ng-required="partition_time.format"
+ ng-class="{ 'col-5': partition_time.type === 'other' }"
+ placeholder="Please enter your custom time format."/>
<span ng-if="state.mode=='view'&&modelsManager.selectedModel.partition_desc.partition_time_column">{{(modelsManager.selectedModel.partition_desc.partition_time_format)}}</span>
</div>
</div>
[kylin] 02/02: KYLIN-3186 expand partition-date-column format
Posted by sh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
shaofengshi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 7f2aa5ad3cbdac59763b7539f1d31bf099d51cb5
Author: chao long <wa...@qq.com>
AuthorDate: Tue Jul 17 19:11:40 2018 +0800
KYLIN-3186 expand partition-date-column format
---
.../org/apache/kylin/common/util/DateFormat.java | 30 +++++++++++++++++++---
.../apache/kylin/metadata/model/PartitionDesc.java | 8 +++---
2 files changed, 31 insertions(+), 7 deletions(-)
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/DateFormat.java b/core-common/src/main/java/org/apache/kylin/common/util/DateFormat.java
index 29858f1..5035e03 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/DateFormat.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/DateFormat.java
@@ -32,6 +32,11 @@ public class DateFormat {
public static final String DEFAULT_TIME_PATTERN = "HH:mm:ss";
public static final String DEFAULT_DATETIME_PATTERN_WITHOUT_MILLISECONDS = "yyyy-MM-dd HH:mm:ss";
public static final String DEFAULT_DATETIME_PATTERN_WITH_MILLISECONDS = "yyyy-MM-dd HH:mm:ss.SSS";
+ public static final String YYYY_MM_DD_HH_MM = "yyyy-MM-dd HH:mm";
+ public static final String YYYY_MM_DD_HH = "yyyy-MM-dd HH";
+ public static final String YYYYMMDDHHMMSS = "yyyyMMddHHmmss";
+ public static final String YYYYMMDDHHMM = "yyyyMMddHHmm";
+ public static final String YYYYMMDDHH = "yyyyMMddHH";
public static final String[] SUPPORTED_DATETIME_PATTERN = { //
DEFAULT_DATE_PATTERN, //
DEFAULT_DATETIME_PATTERN_WITHOUT_MILLISECONDS, //
@@ -48,7 +53,7 @@ public class DateFormat {
}
return r;
}
-
+
public static String formatToCompactDateStr(long millis) {
return formatToDateStr(millis, COMPACT_DATE_PATTERN);
}
@@ -94,13 +99,23 @@ public class DateFormat {
public static long stringToMillis(String str) {
// try to be smart and guess the date format
if (isAllDigits(str)) {
- if (str.length() == 8)
+ if (str.length() == 8 && isInputFormatDate(str, COMPACT_DATE_PATTERN))
//TODO: might be prolematic if an actual ts happends to be 8 digits, e.g. 1970-01-01 10:00:01.123
return stringToDate(str, COMPACT_DATE_PATTERN).getTime();
+ else if (str.length() == 10 && isInputFormatDate(str, YYYYMMDDHH))
+ return stringToDate(str, YYYYMMDDHH).getTime();
+ else if (str.length() == 12 && isInputFormatDate(str, YYYYMMDDHHMM))
+ return stringToDate(str, YYYYMMDDHHMM).getTime();
+ else if (str.length() == 14 && isInputFormatDate(str, YYYYMMDDHHMMSS))
+ return stringToDate(str, YYYYMMDDHHMMSS).getTime();
else
return Long.parseLong(str);
} else if (str.length() == 10) {
return stringToDate(str, DEFAULT_DATE_PATTERN).getTime();
+ } else if (str.length() == 13) {
+ return stringToDate(str, YYYY_MM_DD_HH).getTime();
+ } else if (str.length() == 16) {
+ return stringToDate(str, YYYY_MM_DD_HH_MM).getTime();
} else if (str.length() == 19) {
return stringToDate(str, DEFAULT_DATETIME_PATTERN_WITHOUT_MILLISECONDS).getTime();
} else if (str.length() > 19) {
@@ -137,7 +152,16 @@ public class DateFormat {
return false;
}
+ public static boolean isInputFormatDate(String dateStr, String formatStr) {
+ try {
+ return dateStr.equals(dateToString(stringToDate(dateStr, formatStr), formatStr));
+ } catch (Exception ex) {
+ return false;
+ }
+ }
+
public static boolean isDatePattern(String ptn) {
- return COMPACT_DATE_PATTERN.equals(ptn) || DEFAULT_DATE_PATTERN.equals(ptn);
+ return COMPACT_DATE_PATTERN.equals(ptn) || YYYYMMDDHH.equals(ptn) || YYYYMMDDHHMM.equals(ptn)
+ || YYYYMMDDHHMMSS.equals(ptn);
}
}
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java
index 2f23bdc..b3bf6cc 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java
@@ -199,7 +199,7 @@ public class PartitionDesc implements Serializable {
StringBuilder builder = new StringBuilder();
if (partDesc.partitionColumnIsYmdInt()) {
- buildSingleColumnRangeCondAsYmdInt(builder, partitionDateColumn, startInclusive, endExclusive);
+ buildSingleColumnRangeCondAsYmdInt(builder, partitionDateColumn, startInclusive, endExclusive, partDesc.getPartitionDateFormat());
} else if (partDesc.partitionColumnIsTimeMillis()) {
buildSingleColumnRangeCondAsTimeMillis(builder, partitionDateColumn, startInclusive, endExclusive);
} else if (partitionDateColumn != null && partitionTimeColumn == null) {
@@ -225,13 +225,13 @@ public class PartitionDesc implements Serializable {
}
private static void buildSingleColumnRangeCondAsYmdInt(StringBuilder builder, TblColRef partitionColumn,
- long startInclusive, long endExclusive) {
+ long startInclusive, long endExclusive, String partitionColumnDateFormat) {
String partitionColumnName = partitionColumn.getIdentity();
builder.append(partitionColumnName + " >= "
- + DateFormat.formatToDateStr(startInclusive, DateFormat.COMPACT_DATE_PATTERN));
+ + DateFormat.formatToDateStr(startInclusive, partitionColumnDateFormat));
builder.append(" AND ");
builder.append(partitionColumnName + " < "
- + DateFormat.formatToDateStr(endExclusive, DateFormat.COMPACT_DATE_PATTERN));
+ + DateFormat.formatToDateStr(endExclusive, partitionColumnDateFormat));
}
private static void buildSingleColumnRangeCondition(StringBuilder builder, TblColRef partitionColumn,