You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by ni...@apache.org on 2020/02/07 07:04:00 UTC
[kylin] branch 2.6.x updated (1175527 -> ae63d70)
This is an automated email from the ASF dual-hosted git repository.
nic pushed a change to branch 2.6.x
in repository https://gitbox.apache.org/repos/asf/kylin.git.
from 1175527 KYLIN-1716 UI Location Change Bug Fix
new f940526 #4238 Fix kylin_streaming_model broken when changing kylin.source.hive.database-for-flattable to non-default value
new 472f8c2 KYLIN-4280 SegmenPruner support 'OR' and 'NOT'
new b6b2e16 KYLIN-4287 segmenPruner satisfy "IN" filter bug
new 09074e1 KYLIN-4166 Case when return null when SQL no GROUP BY
new 9d2007f KYLIN-4226: Skip current unavailable tables when updating hbase coprocessor
new 6d7cf02 KYLIN-4290 Add file lock to kylin startup script to avoid starting multiple instances on one node
new 51dcec3 KYLIN-4225 close hive session manually
new fe762b1 KYLIN-4304 Project list cannot be correctly sorted by 'Create Time'
new a578105 KYLIN-4252 Fix the error "Cannot read property 'index' of null" in visualization page
new 8f49194 KYLIN-3609 KYLIN-3409 KYLIN-4161 CubeSegmentStatisticsMap exist only on OLAPQuery.EnumeratorTypeEnum.OLAP query type. (#942)
new 786728e KYLIN-4309 Fixed bug that failed to send to one mailbox will caused other mailboxes have no chance to send
new 2634a7e KYLIN-4080 Project schema update event casues error reload NEW DataModelDesc
new bc11001 Collect information of NumberFormatException
new ae63d70 Set repository sites to https
The 14 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
build/bin/kylin.sh | 10 ++
build/bin/sample.sh | 13 +-
.../org/apache/kylin/common/util/MailService.java | 23 ++-
.../apache/kylin/cube/common/SegmentPruner.java | 56 +++----
.../apache/kylin/cube/common/TupleFilterNode.java | 76 +++++++++
.../kylin/cube/common/SegmentPrunerTest.java | 176 +++++++++++++++++++++
.../apache/kylin/metadata/model/DataModelDesc.java | 12 ++
.../kylin/metadata/model/DataModelManager.java | 29 ++--
.../kylin/metrics/lib/impl/hive/HiveProducer.java | 12 +-
pom.xml | 6 +-
.../apache/kylin/query/relnode/OLAPProjectRel.java | 32 ++--
.../org/apache/kylin/query/util/QueryUtil.java | 5 +
.../kylin/rest/controller/ModelController.java | 1 +
.../kylin/rest/metrics/QueryMetricsFacade.java | 47 +++---
.../storage/hbase/util/DeployCoprocessorCLI.java | 7 +
webapp/app/js/controllers/query.js | 4 +-
webapp/app/js/model/projectConfig.js | 2 +-
17 files changed, 403 insertions(+), 108 deletions(-)
create mode 100644 core-cube/src/main/java/org/apache/kylin/cube/common/TupleFilterNode.java
[kylin] 08/14: KYLIN-4304 Project list cannot be correctly sorted
by 'Create Time'
Posted by ni...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
nic pushed a commit to branch 2.6.x
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit fe762b15bb444d8671f66a3d7cb72b3d93eddabd
Author: xiacongling <xi...@xiaomi.com>
AuthorDate: Tue Dec 17 21:44:07 2019 +0800
KYLIN-4304 Project list cannot be correctly sorted by 'Create Time'
---
webapp/app/js/model/projectConfig.js | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/webapp/app/js/model/projectConfig.js b/webapp/app/js/model/projectConfig.js
index 270eb7a..6bd90a9 100644
--- a/webapp/app/js/model/projectConfig.js
+++ b/webapp/app/js/model/projectConfig.js
@@ -21,7 +21,7 @@ KylinApp.constant('projectConfig', {
{attr: 'name', name: 'Name'},
{attr: 'owner', name: 'Owner'},
{attr: 'description', name: 'Description'},
- {attr: 'create_time', name: 'Create Time'}
+ {attr: 'create_time_utc', name: 'Create Time'}
]
});
[kylin] 14/14: Set repository sites to https
Posted by ni...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
nic pushed a commit to branch 2.6.x
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit ae63d708bd4e865475b3ddd8cd2d4c084f4e29d3
Author: nichunen <ni...@apache.org>
AuthorDate: Fri Feb 7 13:35:47 2020 +0800
Set repository sites to https
---
pom.xml | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/pom.xml b/pom.xml
index 96e729e..3438822 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1015,7 +1015,7 @@
<repository>
<id>central</id>
<name>Central Repository</name>
- <url>http://repo.maven.apache.org/maven2</url>
+ <url>https://repo.maven.apache.org/maven2</url>
<layout>default</layout>
<snapshots>
<enabled>false</enabled>
@@ -1024,7 +1024,7 @@
<repository>
<id>conjars</id>
- <url>http://conjars.org/repo/</url>
+ <url>https://conjars.org/repo/</url>
</repository>
<repository>
@@ -1040,7 +1040,7 @@
<repository>
<id>kyligence</id>
<name>Kyligence Repository</name>
- <url>http://repository.kyligence.io/repository/maven-public/
+ <url>https://repository.kyligence.io/repository/maven-public/
</url>
<releases>
<enabled>true</enabled>
[kylin] 04/14: KYLIN-4166 Case when return null when SQL no GROUP BY
Posted by ni...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
nic pushed a commit to branch 2.6.x
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 09074e1a92ec9e493f8a25fa59f35b5f76ac5b4d
Author: yaqian.zhang <59...@qq.com>
AuthorDate: Mon Oct 21 11:36:09 2019 +0800
KYLIN-4166 Case when return null when SQL no GROUP BY
---
.../apache/kylin/query/relnode/OLAPProjectRel.java | 32 ++++++++++++----------
1 file changed, 17 insertions(+), 15 deletions(-)
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPProjectRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPProjectRel.java
index de4b438..5dde54c 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPProjectRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPProjectRel.java
@@ -264,21 +264,23 @@ public class OLAPProjectRel extends Project implements OLAPRel {
}
// replace projects with dynamic fields
- Map<TblColRef, RelDataType> dynFields = this.context.dynamicFields;
- for (TblColRef dynFieldCol : dynFields.keySet()) {
- String replaceFieldName = dynFieldCol.getName();
- int rowIndex = this.columnRowType.getIndexByName(replaceFieldName);
- if (rowIndex >= 0) {
- int inputIndex = inputColumnRowType.getIndexByName(replaceFieldName);
- if (inputIndex >= 0) {
- // field to be replaced
- RelDataType fieldType = dynFields.get(dynFieldCol);
- RelDataTypeField newField = new RelDataTypeFieldImpl(replaceFieldName, rowIndex, fieldType);
- // project to be replaced
- RelDataTypeField inputField = getInput().getRowType().getFieldList().get(inputIndex);
- RexInputRef newFieldRef = new RexInputRef(inputField.getIndex(), inputField.getType());
-
- replaceFieldMap.put(rowIndex, new Pair<RelDataTypeField, RexNode>(newField, newFieldRef));
+ if (this.context.afterAggregate) {
+ Map<TblColRef, RelDataType> dynFields = this.context.dynamicFields;
+ for (TblColRef dynFieldCol : dynFields.keySet()) {
+ String replaceFieldName = dynFieldCol.getName();
+ int rowIndex = this.columnRowType.getIndexByName(replaceFieldName);
+ if (rowIndex >= 0) {
+ int inputIndex = inputColumnRowType.getIndexByName(replaceFieldName);
+ if (inputIndex >= 0) {
+ // field to be replaced
+ RelDataType fieldType = dynFields.get(dynFieldCol);
+ RelDataTypeField newField = new RelDataTypeFieldImpl(replaceFieldName, rowIndex, fieldType);
+ // project to be replaced
+ RelDataTypeField inputField = getInput().getRowType().getFieldList().get(inputIndex);
+ RexInputRef newFieldRef = new RexInputRef(inputField.getIndex(), inputField.getType());
+
+ replaceFieldMap.put(rowIndex, new Pair<RelDataTypeField, RexNode>(newField, newFieldRef));
+ }
}
}
}
[kylin] 09/14: KYLIN-4252 Fix the error "Cannot read property
'index' of null" in visualization page
Posted by ni...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
nic pushed a commit to branch 2.6.x
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit a578105133bed7b80e37f49693ad772cbce96e7d
Author: Kehua Wu <wu...@zte.com.cn>
AuthorDate: Wed Nov 13 17:58:45 2019 +0800
KYLIN-4252 Fix the error "Cannot read property 'index' of null" in visualization page
---
webapp/app/js/controllers/query.js | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/webapp/app/js/controllers/query.js b/webapp/app/js/controllers/query.js
index e2b8520..569bdc5 100644
--- a/webapp/app/js/controllers/query.js
+++ b/webapp/app/js/controllers/query.js
@@ -508,11 +508,11 @@ KylinApp
$scope.chart = undefined;
var selectedDimension = query.graph.state.dimensions;
- if (selectedDimension && query.graph.type.dimension.types.indexOf(selectedDimension.type) > -1) {
+ var selectedMetric = query.graph.state.metrics;
+ if (selectedDimension && selectedMetric && query.graph.type.dimension.types.indexOf(selectedDimension.type) > -1) {
$scope.chart = {};
var chartType = query.graph.type.value;
- var selectedMetric = query.graph.state.metrics;
var dataValues = [];
angular.forEach(query.result.results, function(result, ind) {
[kylin] 01/14: #4238 Fix kylin_streaming_model broken when changing
kylin.source.hive.database-for-flattable to non-default value
Posted by ni...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
nic pushed a commit to branch 2.6.x
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit f94052622d2d6cf747b76ab8b909d9fcaab32b4f
Author: mingxiang.zhao <mz...@gmail.com>
AuthorDate: Tue Nov 5 20:29:54 2019 +0800
#4238 Fix kylin_streaming_model broken when changing kylin.source.hive.database-for-flattable to non-default value
---
build/bin/sample.sh | 13 +++++++++++--
1 file changed, 11 insertions(+), 2 deletions(-)
diff --git a/build/bin/sample.sh b/build/bin/sample.sh
index 18823ae..09ca710 100755
--- a/build/bin/sample.sh
+++ b/build/bin/sample.sh
@@ -104,8 +104,17 @@ sed -i "s/DEFAULT./$sample_database./g" ${KYLIN_HOME}/sample_cube/metadata/model
sed -i "s/DEFAULT./$sample_database./g" ${KYLIN_HOME}/sample_cube/metadata/model_desc/kylin_streaming_model.json
sed -i "s/DEFAULT./$sample_database./g" ${KYLIN_HOME}/sample_cube/metadata/project/learn_kylin.json
sed -i "s/DEFAULT/$sample_database/g" ${KYLIN_HOME}/sample_cube/metadata/table/*.json
-cd ${KYLIN_HOME}/sample_cube/metadata/table
-ls -1 DEFAULT.KYLIN_*.json|sed "s/\(DEFAULT\)\(.*\)\.json/mv & $sample_database\2.json/"|sh -v
+sed -i "s/DEFAULT/$sample_database/g" ${KYLIN_HOME}/sample_cube/metadata/kafka/*.json
+sed -i "s/DEFAULT/$sample_database/g" ${KYLIN_HOME}/sample_cube/metadata/streaming/*.json
+strings=(
+ table
+ streaming
+ kafka
+)
+for i in "${strings[@]}"; do
+ cd "${KYLIN_HOME}/sample_cube/metadata/${i}"
+ ls -1 DEFAULT.KYLIN_*.json|sed "s/\(DEFAULT\)\(.*\)\.json/mv & $sample_database\2.json/"|sh -v
+done
cd ${KYLIN_HOME}
${dir}/kylin.sh org.apache.kylin.common.persistence.ResourceTool upload ${KYLIN_HOME}/sample_cube/metadata || { exit 1; }
[kylin] 13/14: Collect information of NumberFormatException
Posted by ni...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
nic pushed a commit to branch 2.6.x
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit bc11001208875f1f351c36634ac60868a2026822
Author: baiyunsong <ba...@xiaomi.com>
AuthorDate: Thu Jan 2 11:52:13 2020 +0800
Collect information of NumberFormatException
---
query/src/main/java/org/apache/kylin/query/util/QueryUtil.java | 5 +++++
1 file changed, 5 insertions(+)
diff --git a/query/src/main/java/org/apache/kylin/query/util/QueryUtil.java b/query/src/main/java/org/apache/kylin/query/util/QueryUtil.java
index 6407308..0171432 100644
--- a/query/src/main/java/org/apache/kylin/query/util/QueryUtil.java
+++ b/query/src/main/java/org/apache/kylin/query/util/QueryUtil.java
@@ -152,6 +152,11 @@ public class QueryUtil {
msg = "ArithmeticException: " + cause.getMessage();
break;
}
+
+ if (cause.getClass().getName().contains("NumberFormatException")) {
+ msg = "NumberFormatException: " + cause.getMessage();
+ break;
+ }
cause = cause.getCause();
}
[kylin] 12/14: KYLIN-4080 Project schema update event casues error
reload NEW DataModelDesc
Posted by ni...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
nic pushed a commit to branch 2.6.x
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 2634a7ec01fbc426ba3d21b708c7b13a3c07398b
Author: yaqian.zhang <59...@qq.com>
AuthorDate: Wed Dec 4 19:50:38 2019 +0800
KYLIN-4080 Project schema update event casues error reload NEW DataModelDesc
---
.../apache/kylin/metadata/model/DataModelDesc.java | 12 +++++++++
.../kylin/metadata/model/DataModelManager.java | 29 ++++++++--------------
.../kylin/rest/controller/ModelController.java | 1 +
3 files changed, 23 insertions(+), 19 deletions(-)
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
index 5795d78..818afdf 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
@@ -104,6 +104,9 @@ public class DataModelDesc extends RootPersistentEntity {
@JsonProperty("capacity")
private RealizationCapacity capacity = RealizationCapacity.MEDIUM;
+ @JsonProperty("projectName")
+ private String projectName; //for KYLIN-4080
+
// computed attributes
private TableRef rootFactTableRef;
private Set<TableRef> factTableRefs = Sets.newLinkedHashSet();
@@ -765,6 +768,14 @@ public class DataModelDesc extends RootPersistentEntity {
return ProjectManager.getInstance(getConfig()).getProjectOfModel(this.getName());
}
+ public String getProjectName() {
+ return projectName;
+ }
+
+ public void setProjectName(String projectName) {
+ this.projectName = projectName;
+ }
+
public static DataModelDesc getCopyOf(DataModelDesc orig) {
return copy(orig, new DataModelDesc());
}
@@ -781,6 +792,7 @@ public class DataModelDesc extends RootPersistentEntity {
copy.metrics = orig.metrics;
copy.filterCondition = orig.filterCondition;
copy.capacity = orig.capacity;
+ copy.projectName = orig.projectName;
if (orig.getPartitionDesc() != null) {
copy.partitionDesc = PartitionDesc.getCopyOf(orig.getPartitionDesc());
}
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelManager.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelManager.java
index 88ba93d..1022451 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelManager.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelManager.java
@@ -86,7 +86,9 @@ public class DataModelManager {
getDataModelImplClass(), dataModelDescMap) {
@Override
protected DataModelDesc initEntityAfterReload(DataModelDesc model, String resourceName) {
- String prj = ProjectManager.getInstance(config).getProjectOfModel(model.getName()).getName();
+ String prj = (null == model.getProjectName()
+ ? ProjectManager.getInstance(config).getProjectOfModel(model.getName()).getName()
+ : model.getProjectName());
if (!model.isDraft()) {
model.init(config, getAllTablesMap(prj));
}
@@ -236,21 +238,12 @@ public class DataModelManager {
ProjectManager prjMgr = ProjectManager.getInstance(config);
ProjectInstance prj = prjMgr.getProject(projectName);
- if (prj.containsModel(name))
+ if (prj.containsModel(name)) {
throw new IllegalStateException("project " + projectName + " already contains model " + name);
-
- try {
- // Temporarily register model under project, because we want to
- // update project formally after model is saved.
- prj.getModels().add(name);
-
- desc.setOwner(owner);
- logger.info("Saving Model {} to Project {} with {} as owner", desc.getName(), projectName, owner);
- desc = saveDataModelDesc(desc);
-
- } finally {
- prj.getModels().remove(name);
}
+ desc.setOwner(owner);
+ logger.info("Saving Model {} to Project {} with {} as owner", desc.getName(), projectName, owner);
+ desc = saveDataModelDesc(desc, projectName);
// now that model is saved, update project formally
prjMgr.addModelToProject(name, projectName);
@@ -266,16 +259,14 @@ public class DataModelManager {
throw new IllegalArgumentException("DataModelDesc '" + name + "' does not exist.");
}
- return saveDataModelDesc(desc);
+ return saveDataModelDesc(desc, ProjectManager.getInstance(config).getProjectOfModel(desc.getName()).getName());
}
}
- private DataModelDesc saveDataModelDesc(DataModelDesc dataModelDesc) throws IOException {
-
- String prj = ProjectManager.getInstance(config).getProjectOfModel(dataModelDesc.getName()).getName();
+ private DataModelDesc saveDataModelDesc(DataModelDesc dataModelDesc, String projectName) throws IOException {
if (!dataModelDesc.isDraft())
- dataModelDesc.init(config, this.getAllTablesMap(prj));
+ dataModelDesc.init(config, this.getAllTablesMap(projectName));
crud.save(dataModelDesc);
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/ModelController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/ModelController.java
index 820d422..5ffb51c 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller/ModelController.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller/ModelController.java
@@ -228,6 +228,7 @@ public class ModelController extends BasicController {
try {
logger.debug("Saving MODEL " + modelRequest.getModelDescData());
desc = JsonUtil.readValue(modelRequest.getModelDescData(), DataModelDesc.class);
+ desc.setProjectName(modelRequest.getProject());
} catch (JsonParseException e) {
logger.error("The data model definition is not valid.", e);
updateRequest(modelRequest, false, e.getMessage());
[kylin] 10/14: KYLIN-3609 KYLIN-3409 KYLIN-4161
CubeSegmentStatisticsMap exist only on OLAPQuery.EnumeratorTypeEnum.OLAP
query type. (#942)
Posted by ni...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
nic pushed a commit to branch 2.6.x
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 8f491940429704500ec9628569d5c90d32d73642
Author: Alexander Sidorchuk <39...@users.noreply.github.com>
AuthorDate: Mon Dec 23 16:17:06 2019 +0300
KYLIN-3609 KYLIN-3409 KYLIN-4161 CubeSegmentStatisticsMap exist only on OLAPQuery.EnumeratorTypeEnum.OLAP query type. (#942)
---
.../kylin/rest/metrics/QueryMetricsFacade.java | 47 +++++++++++++---------
1 file changed, 27 insertions(+), 20 deletions(-)
diff --git a/server-base/src/main/java/org/apache/kylin/rest/metrics/QueryMetricsFacade.java b/server-base/src/main/java/org/apache/kylin/rest/metrics/QueryMetricsFacade.java
index 497f477..226166d 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/metrics/QueryMetricsFacade.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/metrics/QueryMetricsFacade.java
@@ -36,6 +36,7 @@ import org.apache.kylin.metrics.lib.impl.TimedRecordEvent;
import org.apache.kylin.metrics.property.QueryCubePropertyEnum;
import org.apache.kylin.metrics.property.QueryPropertyEnum;
import org.apache.kylin.metrics.property.QueryRPCPropertyEnum;
+import org.apache.kylin.query.enumerator.OLAPQuery;
import org.apache.kylin.rest.request.SQLRequest;
import org.apache.kylin.rest.response.SQLResponse;
import org.slf4j.Logger;
@@ -121,26 +122,32 @@ public class QueryMetricsFacade {
sqlResponse.getThrowable());
long totalStorageReturnCount = 0L;
- for (Map<String, QueryContext.CubeSegmentStatistics> cubeEntry : contextEntry.getCubeSegmentStatisticsMap()
- .values()) {
- for (QueryContext.CubeSegmentStatistics segmentEntry : cubeEntry.values()) {
- RecordEvent cubeSegmentMetricsEvent = new TimedRecordEvent(
- KylinConfig.getInstanceFromEnv().getKylinMetricsSubjectQueryCube());
-
- setCubeWrapper(cubeSegmentMetricsEvent, //
- norm(sqlRequest.getProject()), segmentEntry.getCubeName(), segmentEntry.getSegmentName(),
- segmentEntry.getSourceCuboidId(), segmentEntry.getTargetCuboidId(),
- segmentEntry.getFilterMask());
-
- setCubeStats(cubeSegmentMetricsEvent, //
- segmentEntry.getCallCount(), segmentEntry.getCallTimeSum(), segmentEntry.getCallTimeMax(),
- segmentEntry.getStorageSkippedRows(), segmentEntry.getStorageScannedRows(),
- segmentEntry.getStorageReturnedRows(), segmentEntry.getStorageAggregatedRows(),
- segmentEntry.isIfSuccess(), 1.0 / cubeEntry.size());
-
- totalStorageReturnCount += segmentEntry.getStorageReturnedRows();
- //For update cube segment level related query metrics
- MetricsManager.getInstance().update(cubeSegmentMetricsEvent);
+ if (contextEntry.getQueryType().equalsIgnoreCase(OLAPQuery.EnumeratorTypeEnum.OLAP.name())) {
+ for (Map<String, QueryContext.CubeSegmentStatistics> cubeEntry : contextEntry.getCubeSegmentStatisticsMap()
+ .values()) {
+ for (QueryContext.CubeSegmentStatistics segmentEntry : cubeEntry.values()) {
+ RecordEvent cubeSegmentMetricsEvent = new TimedRecordEvent(
+ KylinConfig.getInstanceFromEnv().getKylinMetricsSubjectQueryCube());
+
+ setCubeWrapper(cubeSegmentMetricsEvent, //
+ norm(sqlRequest.getProject()), segmentEntry.getCubeName(), segmentEntry.getSegmentName(),
+ segmentEntry.getSourceCuboidId(), segmentEntry.getTargetCuboidId(),
+ segmentEntry.getFilterMask());
+
+ setCubeStats(cubeSegmentMetricsEvent, //
+ segmentEntry.getCallCount(), segmentEntry.getCallTimeSum(), segmentEntry.getCallTimeMax(),
+ segmentEntry.getStorageSkippedRows(), segmentEntry.getStorageScannedRows(),
+ segmentEntry.getStorageReturnedRows(), segmentEntry.getStorageAggregatedRows(),
+ segmentEntry.isIfSuccess(), 1.0 / cubeEntry.size());
+
+ totalStorageReturnCount += segmentEntry.getStorageReturnedRows();
+ //For update cube segment level related query metrics
+ MetricsManager.getInstance().update(cubeSegmentMetricsEvent);
+ }
+ }
+ } else {
+ if (!sqlResponse.getIsException()) {
+ totalStorageReturnCount = sqlResponse.getResults().size();
}
}
setQueryStats(queryMetricsEvent, //
[kylin] 07/14: KYLIN-4225 close hive session manually
Posted by ni...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
nic pushed a commit to branch 2.6.x
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 51dcec3507e761c6346a20af88b3f23c35843743
Author: Zhou Kang <zh...@xiaomi.com>
AuthorDate: Wed Oct 30 10:14:57 2019 +0800
KYLIN-4225 close hive session manually
---
.../org/apache/kylin/metrics/lib/impl/hive/HiveProducer.java | 12 ++++++++++--
1 file changed, 10 insertions(+), 2 deletions(-)
diff --git a/metrics-reporter-hive/src/main/java/org/apache/kylin/metrics/lib/impl/hive/HiveProducer.java b/metrics-reporter-hive/src/main/java/org/apache/kylin/metrics/lib/impl/hive/HiveProducer.java
index 04a1f20..8062b12 100644
--- a/metrics-reporter-hive/src/main/java/org/apache/kylin/metrics/lib/impl/hive/HiveProducer.java
+++ b/metrics-reporter-hive/src/main/java/org/apache/kylin/metrics/lib/impl/hive/HiveProducer.java
@@ -34,6 +34,7 @@ import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.ql.Driver;
+import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse;
import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.kylin.common.util.Pair;
import org.apache.kylin.metrics.lib.ActiveReservoirReporter;
@@ -151,8 +152,15 @@ public class HiveProducer {
}
hql.append(")");
Driver driver = new Driver(hiveConf);
- SessionState.start(new CliSessionState(hiveConf));
- driver.run(hql.toString());
+ CliSessionState session = new CliSessionState(hiveConf);
+ SessionState.start(session);
+ CommandProcessorResponse res = driver.run(hql.toString());
+ if (res.getResponseCode() != 0) {
+ logger.warn("Fail to add partition. HQL: {}; Cause by: {}",
+ hql.toString(),
+ res.toString());
+ }
+ session.close();
driver.close();
}
Path partitionContentPath = new Path(partitionPath, CONTENT_FILE_NAME);
[kylin] 02/14: KYLIN-4280 SegmenPruner support 'OR' and 'NOT'
Posted by ni...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
nic pushed a commit to branch 2.6.x
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 472f8c228ec1f24eae48356a20bd836f038e826b
Author: zhangbushi5 <84...@qq.com>
AuthorDate: Fri Dec 6 17:04:22 2019 +0800
KYLIN-4280 SegmenPruner support 'OR' and 'NOT'
---
.../apache/kylin/cube/common/SegmentPruner.java | 48 ++-----
.../apache/kylin/cube/common/TupleFilterNode.java | 76 +++++++++++
.../kylin/cube/common/SegmentPrunerTest.java | 151 +++++++++++++++++++++
3 files changed, 241 insertions(+), 34 deletions(-)
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/common/SegmentPruner.java b/core-cube/src/main/java/org/apache/kylin/cube/common/SegmentPruner.java
index f3f2052..ae21a4d 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/common/SegmentPruner.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/common/SegmentPruner.java
@@ -19,9 +19,7 @@
package org.apache.kylin.cube.common;
import java.util.ArrayList;
-import java.util.Collections;
import java.util.List;
-import java.util.Map;
import java.util.Set;
import org.apache.commons.lang.StringUtils;
@@ -45,11 +43,11 @@ import org.slf4j.LoggerFactory;
public class SegmentPruner {
private static final Logger logger = LoggerFactory.getLogger(SegmentPruner.class);
- final private Set<CompareTupleFilter> mustTrueCompares;
+ private TupleFilterNode node;
public SegmentPruner(TupleFilter filter) {
- this.mustTrueCompares = filter == null ? Collections.<CompareTupleFilter> emptySet()
- : filter.findMustTrueCompareFilters();
+ this.node = new TupleFilterNode(filter);
+
}
public List<CubeSegment> listSegmentsForQuery(CubeInstance cube) {
@@ -62,7 +60,7 @@ public class SegmentPruner {
}
public boolean check(CubeSegment seg) {
-
+
if (seg.getInputRecords() == 0) {
if (seg.getConfig().isSkippingEmptySegments()) {
logger.debug("Prune segment {} due to 0 input record", seg);
@@ -72,53 +70,35 @@ public class SegmentPruner {
}
}
- Map<String, DimensionRangeInfo> segDimRangInfoMap = seg.getDimensionRangeInfoMap();
- for (CompareTupleFilter comp : mustTrueCompares) {
- TblColRef col = comp.getColumn();
-
- if (!col.getType().needCompare()) {
- continue;
- }
-
- DimensionRangeInfo dimRangeInfo = segDimRangInfoMap.get(col.getIdentity());
- if (dimRangeInfo == null)
- dimRangeInfo = tryDeduceRangeFromPartitionCol(seg, col);
- if (dimRangeInfo == null)
- continue;
-
- String minVal = dimRangeInfo.getMin();
- String maxVal = dimRangeInfo.getMax();
-
- if (!satisfy(comp, minVal, maxVal)) {
- logger.debug("Prune segment {} due to given filter", seg);
- return false;
- }
+ if (!node.checkSeg(seg)) {
+ logger.debug("Prune segment {} due to given filter", seg);
+ return false;
}
logger.debug("Pruner passed on segment {}", seg);
return true;
}
- private DimensionRangeInfo tryDeduceRangeFromPartitionCol(CubeSegment seg, TblColRef col) {
+ public static DimensionRangeInfo tryDeduceRangeFromPartitionCol(CubeSegment seg, TblColRef col) {
DataModelDesc model = seg.getModel();
PartitionDesc part = model.getPartitionDesc();
-
+
if (!part.isPartitioned())
return null;
if (!col.equals(part.getPartitionDateColumnRef()))
return null;
-
+
// deduce the dim range from TSRange
TSRange tsRange = seg.getTSRange();
if (tsRange.start.isMin || tsRange.end.isMax)
return null; // DimensionRangeInfo cannot express infinite
-
+
String min = tsRangeToStr(tsRange.start.v, part);
String max = tsRangeToStr(tsRange.end.v - 1, part); // note the -1, end side is exclusive
return new DimensionRangeInfo(min, max);
}
- private String tsRangeToStr(long ts, PartitionDesc part) {
+ private static String tsRangeToStr(long ts, PartitionDesc part) {
String value;
DataType partitionColType = part.getPartitionDateColumnRef().getType();
if (partitionColType.isDate()) {
@@ -138,7 +118,7 @@ public class SegmentPruner {
return value;
}
- private boolean satisfy(CompareTupleFilter comp, String minVal, String maxVal) {
+ public static boolean satisfy(CompareTupleFilter comp, String minVal, String maxVal) {
// When both min and max are null, it means all cells of the column are null.
// In such case, return true to let query engine scan the segment, since the
@@ -177,7 +157,7 @@ public class SegmentPruner {
}
}
- private String toString(Object v) {
+ private static String toString(Object v) {
return v == null ? null : v.toString();
}
}
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/common/TupleFilterNode.java b/core-cube/src/main/java/org/apache/kylin/cube/common/TupleFilterNode.java
new file mode 100644
index 0000000..8a84f67
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/cube/common/TupleFilterNode.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.cube.common;
+
+import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.cube.DimensionRangeInfo;
+import org.apache.kylin.metadata.filter.CompareTupleFilter;
+import org.apache.kylin.metadata.filter.LogicalTupleFilter;
+import org.apache.kylin.metadata.filter.TupleFilter;
+import org.apache.kylin.metadata.model.TblColRef;
+
+public class TupleFilterNode {
+ private TupleFilter filter;
+
+ public TupleFilterNode(TupleFilter filter) {
+ this.filter = filter;
+ }
+
+ public boolean checkSeg(CubeSegment seg) {
+ if (filter == null)
+ return true;
+ if (filter instanceof CompareTupleFilter) {
+ CompareTupleFilter compareTupleFilter = (CompareTupleFilter) filter;
+ TblColRef col = compareTupleFilter.getColumn();
+ if (col == null) {
+ return true;
+ }
+ DimensionRangeInfo dimRangeInfo = seg.getDimensionRangeInfoMap().get(col.getIdentity());
+ if (dimRangeInfo == null)
+ dimRangeInfo = SegmentPruner.tryDeduceRangeFromPartitionCol(seg, col);
+ if (dimRangeInfo == null)
+ return true;
+ String minVal = dimRangeInfo.getMin();
+ String maxVal = dimRangeInfo.getMax();
+ return SegmentPruner.satisfy(compareTupleFilter, minVal, maxVal);
+ }
+
+ if (filter instanceof LogicalTupleFilter) {
+ if (filter.getOperator() == TupleFilter.FilterOperatorEnum.AND) {
+ for (TupleFilter filter : filter.getChildren()) {
+ if (!new TupleFilterNode(filter).checkSeg(seg))
+ return false;
+ }
+ return true;
+
+ } else if (filter.getOperator() == TupleFilter.FilterOperatorEnum.OR) {
+ for (TupleFilter filter : filter.getChildren()) {
+ if (new TupleFilterNode(filter).checkSeg(seg))
+ return true;
+ }
+ return false;
+ } else if (filter.getOperator() == TupleFilter.FilterOperatorEnum.NOT) {
+ for (TupleFilter filter : filter.getChildren()) {
+ return !(new TupleFilterNode(filter).checkSeg(seg));
+ }
+ }
+ }
+ return true;
+ }
+}
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/common/SegmentPrunerTest.java b/core-cube/src/test/java/org/apache/kylin/cube/common/SegmentPrunerTest.java
index 603db97..69f9b10 100644
--- a/core-cube/src/test/java/org/apache/kylin/cube/common/SegmentPrunerTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/cube/common/SegmentPrunerTest.java
@@ -18,15 +18,21 @@
package org.apache.kylin.cube.common;
+import static org.apache.kylin.metadata.filter.TupleFilter.and;
import static org.apache.kylin.metadata.filter.TupleFilter.compare;
+import static org.apache.kylin.metadata.filter.TupleFilter.or;
+import com.google.common.collect.Maps;
import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.DateFormat;
import org.apache.kylin.common.util.LocalFileMetadataTestCase;
import org.apache.kylin.common.util.SetAndUnsetSystemProp;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.cube.DimensionRangeInfo;
import org.apache.kylin.metadata.filter.ConstantTupleFilter;
+import org.apache.kylin.metadata.filter.LogicalTupleFilter;
import org.apache.kylin.metadata.filter.TupleFilter;
import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum;
import org.apache.kylin.metadata.model.SegmentRange.TSRange;
@@ -39,6 +45,8 @@ import org.junit.Test;
import com.google.common.collect.Sets;
+import java.util.Map;
+
public class SegmentPrunerTest extends LocalFileMetadataTestCase {
private CubeInstance cube;
@@ -192,4 +200,147 @@ public class SegmentPrunerTest extends LocalFileMetadataTestCase {
}
}
}
+
+ @Test
+ public void testLegacyCubeSegWithOrFilter() {
+ // legacy cube segments does not have DimensionRangeInfo, but with TSRange can do some pruning
+ CubeInstance cube = CubeManager.getInstance(getTestConfig())
+ .getCube("test_kylin_cube_without_slr_left_join_ready_2_segments");
+
+ TblColRef col = cube.getModel().findColumn("TEST_KYLIN_FACT.CAL_DT");
+ TblColRef col2 = cube.getModel().findColumn("TEST_KYLIN_FACT.LSTG_SITE_ID");
+ CubeSegment seg = cube.getSegments(SegmentStatusEnum.READY).get(0);
+ Map<String, DimensionRangeInfo> dimensionRangeInfoMap = Maps.newHashMap();
+ dimensionRangeInfoMap.put("TEST_KYLIN_FACT.LSTG_SITE_ID", new DimensionRangeInfo("10", "20"));
+ seg.setDimensionRangeInfoMap(dimensionRangeInfoMap);
+ TSRange tsRange = seg.getTSRange();
+ String start = DateFormat.formatToTimeStr(tsRange.start.v, "yyyy-MM-dd");
+
+ CubeSegment seg2 = cube.getSegments(SegmentStatusEnum.READY).get(1);
+ Map<String, DimensionRangeInfo> dimensionRangeInfoMap2 = Maps.newHashMap();
+ dimensionRangeInfoMap2.put("TEST_KYLIN_FACT.LSTG_SITE_ID", new DimensionRangeInfo("20", "30"));
+ seg2.setDimensionRangeInfoMap(dimensionRangeInfoMap2);
+ TSRange tsRange2 = seg2.getTSRange();
+ String start2 = DateFormat.formatToTimeStr(tsRange2.start.v, "yyyy-MM-dd");
+
+ try (SetAndUnsetSystemProp sns = new SetAndUnsetSystemProp("kylin.query.skip-empty-segments", "false")) {
+ {
+ TupleFilter andFilter1 = compare(col, FilterOperatorEnum.EQ, start);
+ TupleFilter andFilter2 = compare(col2, FilterOperatorEnum.EQ, "15");
+ LogicalTupleFilter logicalAndFilter = and(andFilter1, andFilter2);
+
+ TupleFilter andFilter3 = compare(col, FilterOperatorEnum.EQ, start2);
+ TupleFilter andFilter4 = compare(col2, FilterOperatorEnum.EQ, "25");
+ LogicalTupleFilter logicalAndFilter2 = and(andFilter3, andFilter4);
+
+ LogicalTupleFilter finalFilter = or(logicalAndFilter, logicalAndFilter2);
+ SegmentPruner segmentPruner = new SegmentPruner(finalFilter);
+ Assert.assertTrue(segmentPruner.check(seg));
+ Assert.assertTrue(segmentPruner.check(seg2));
+ }
+
+ {
+ TupleFilter andFilter1 = compare(col, FilterOperatorEnum.EQ, start);
+ TupleFilter andFilter2 = compare(col2, FilterOperatorEnum.EQ, "15");
+ LogicalTupleFilter logicalAndFilter = and(andFilter1, andFilter2);
+
+ TupleFilter andFilter3 = compare(col, FilterOperatorEnum.EQ, start2);
+ TupleFilter andFilter4 = compare(col2, FilterOperatorEnum.EQ, "35");
+ LogicalTupleFilter logicalAndFilter2 = and(andFilter3, andFilter4);
+
+ LogicalTupleFilter finalFilter = or(logicalAndFilter, logicalAndFilter2);
+ SegmentPruner segmentPruner = new SegmentPruner(finalFilter);
+ Assert.assertTrue(segmentPruner.check(seg));
+ Assert.assertFalse(segmentPruner.check(seg2));
+ }
+
+ {
+ TupleFilter andFilter1 = compare(col, FilterOperatorEnum.EQ, start);
+ TupleFilter andFilter2 = compare(col2, FilterOperatorEnum.EQ, "15");
+ LogicalTupleFilter logicalAndFilter = and(andFilter1, andFilter2);
+
+ TupleFilter andFilter3 = compare(col, FilterOperatorEnum.EQ, start2);
+ TupleFilter andFilter4 = compare(col2, FilterOperatorEnum.EQ, "35");
+ LogicalTupleFilter logicalAndFilter2 = and(andFilter3, andFilter4);
+
+ LogicalTupleFilter finalFilter = and(logicalAndFilter, logicalAndFilter2);
+ SegmentPruner segmentPruner = new SegmentPruner(finalFilter);
+ Assert.assertFalse(segmentPruner.check(seg));
+ Assert.assertFalse(segmentPruner.check(seg2));
+ }
+
+ {
+ TupleFilter andFilter1 = compare(col, FilterOperatorEnum.EQ, start);
+ TupleFilter andFilter2 = compare(col2, FilterOperatorEnum.EQ, "15");
+ LogicalTupleFilter logicalAndFilter = and(andFilter1, andFilter2);
+
+ TupleFilter andFilter3 = compare(col, FilterOperatorEnum.LT, start2);
+ TupleFilter andFilter4 = compare(col2, FilterOperatorEnum.LT, "35");
+ LogicalTupleFilter logicalAndFilter2 = and(andFilter3, andFilter4);
+
+ LogicalTupleFilter finalFilter = and(logicalAndFilter, logicalAndFilter2);
+ SegmentPruner segmentPruner = new SegmentPruner(finalFilter);
+ Assert.assertTrue(segmentPruner.check(seg));
+ Assert.assertFalse(segmentPruner.check(seg2));
+ }
+
+ {
+ TupleFilter andFilter1 = compare(col, FilterOperatorEnum.EQ, start);
+ TupleFilter andFilter2 = compare(col2, FilterOperatorEnum.EQ, "35");
+ LogicalTupleFilter logicalAndFilter = or(andFilter1, andFilter2);
+
+ TupleFilter andFilter3 = compare(col, FilterOperatorEnum.EQ, start2);
+ TupleFilter andFilter4 = compare(col2, FilterOperatorEnum.LT, "35");
+ LogicalTupleFilter logicalAndFilter2 = or(andFilter3, andFilter4);
+
+ LogicalTupleFilter finalFilter = or(logicalAndFilter, logicalAndFilter2);
+ SegmentPruner segmentPruner = new SegmentPruner(finalFilter);
+ Assert.assertTrue(segmentPruner.check(seg));
+ Assert.assertTrue(segmentPruner.check(seg2));
+ }
+
+ {
+ TupleFilter andFilter1 = compare(col, FilterOperatorEnum.EQ, start);
+ TupleFilter andFilter2 = compare(col2, FilterOperatorEnum.IN, "15");
+ LogicalTupleFilter logicalAndFilter = and(andFilter1, andFilter2);
+
+ TupleFilter andFilter3 = compare(col, FilterOperatorEnum.EQ, start2);
+ TupleFilter andFilter4 = compare(col2, FilterOperatorEnum.LT, "35");
+ LogicalTupleFilter logicalAndFilter2 = or(andFilter3, andFilter4);
+
+ LogicalTupleFilter finalFilter = or(logicalAndFilter, logicalAndFilter2);
+ SegmentPruner segmentPruner = new SegmentPruner(finalFilter);
+ Assert.assertTrue(segmentPruner.check(seg));
+ Assert.assertTrue(segmentPruner.check(seg2));
+ }
+
+ {
+ TupleFilter andFilter1 = compare(col, FilterOperatorEnum.EQ, start);
+ TupleFilter andFilter2 = compare(col2, FilterOperatorEnum.EQ, "35");
+ LogicalTupleFilter logicalAndFilter = or(andFilter1, andFilter2);
+
+ TupleFilter andFilter3 = compare(col, FilterOperatorEnum.EQ, start2);
+ TupleFilter andFilter4 = compare(col2, FilterOperatorEnum.LT, "35");
+ LogicalTupleFilter logicalAndFilter2 = or(andFilter3, andFilter4);
+
+ LogicalTupleFilter finalFilter1 = or(logicalAndFilter, logicalAndFilter2);
+
+ TupleFilter andFilter5 = compare(col, FilterOperatorEnum.EQ, start);
+ TupleFilter andFilter6 = compare(col2, FilterOperatorEnum.IN, "15");
+ LogicalTupleFilter logicalAndFilter3 = and(andFilter5, andFilter6);
+
+ TupleFilter andFilter7 = compare(col, FilterOperatorEnum.EQ, start2);
+ TupleFilter andFilter8 = compare(col2, FilterOperatorEnum.LT, "35");
+ LogicalTupleFilter logicalAndFilter4 = or(andFilter7, andFilter8);
+
+ LogicalTupleFilter finalFilter2 = or(logicalAndFilter3, logicalAndFilter4);
+
+ LogicalTupleFilter finalFinalFilter = or(finalFilter1, finalFilter2);
+
+ SegmentPruner segmentPruner = new SegmentPruner(finalFinalFilter);
+ Assert.assertTrue(segmentPruner.check(seg));
+ Assert.assertTrue(segmentPruner.check(seg2));
+ }
+ }
+ }
}
[kylin] 11/14: KYLIN-4309 Fixed bug that failed to send to one
mailbox will caused other mailboxes have no chance to send
Posted by ni...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
nic pushed a commit to branch 2.6.x
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 786728eb8710145916f100dfa4e8b7b60e4e649f
Author: weibin0516 <co...@126.com>
AuthorDate: Fri Dec 20 13:22:31 2019 +0800
KYLIN-4309 Fixed bug that failed to send to one mailbox will caused other mailboxes have no chance to send
---
.../org/apache/kylin/common/util/MailService.java | 23 +++++++++++++++-------
1 file changed, 16 insertions(+), 7 deletions(-)
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/MailService.java b/core-common/src/main/java/org/apache/kylin/common/util/MailService.java
index 44e1d9c..b68c78d 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/MailService.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/MailService.java
@@ -22,7 +22,6 @@ import java.io.IOException;
import java.util.List;
import org.apache.commons.mail.Email;
-import org.apache.commons.mail.EmailException;
import org.apache.commons.mail.HtmlEmail;
import org.apache.kylin.common.KylinConfig;
import org.slf4j.LoggerFactory;
@@ -100,12 +99,23 @@ public class MailService {
email.setAuthentication(username, password);
}
- //email.setDebug(true);
- try {
- for (String receiver : receivers) {
+ for (String receiver : receivers) {
+ try {
email.addTo(receiver);
+ } catch (Exception e) {
+ logger.error("add " + receiver + " to send to mailbox list failed, " +
+ "this will not affect sending to the valid mailbox", e);
}
+ }
+ // List of valid recipients is empty
+ if (email.getToAddresses().isEmpty()) {
+ logger.error("No valid send to mailbox, please check");
+ return false;
+ }
+
+ // List of valid recipients is not empty
+ try {
email.setFrom(sender);
email.setSubject(subject);
email.setCharset("UTF-8");
@@ -117,11 +127,10 @@ public class MailService {
email.send();
email.getMailSession();
- } catch (EmailException e) {
+ return true;
+ } catch (Exception e) {
logger.error(e.getLocalizedMessage(), e);
return false;
}
-
- return true;
}
}
[kylin] 06/14: KYLIN-4290 Add file lock to kylin startup script to
avoid starting multiple instances on one node
Posted by ni...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
nic pushed a commit to branch 2.6.x
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 6d7cf0220a6dd24ea5661a973c475f6e552fbba9
Author: yangzheng10 <ya...@58.com>
AuthorDate: Tue Dec 10 19:10:51 2019 +0800
KYLIN-4290 Add file lock to kylin startup script to avoid starting multiple instances on one node
---
build/bin/kylin.sh | 10 ++++++++++
1 file changed, 10 insertions(+)
diff --git a/build/bin/kylin.sh b/build/bin/kylin.sh
index d6a311e..00e545a 100755
--- a/build/bin/kylin.sh
+++ b/build/bin/kylin.sh
@@ -66,6 +66,14 @@ function retrieveStartCommand() {
fi
fi
+ lockfile=$KYLIN_HOME/LOCK
+ if [ ! -e $lockfile ]; then
+ trap "rm -f $lockfile; exit" INT TERM EXIT
+ touch $lockfile
+ else
+ quit "Kylin is starting, wait for it"
+ fi
+
source ${dir}/check-env.sh
tomcat_root=${dir}/../tomcat
@@ -130,6 +138,7 @@ if [ "$1" == "start" ]
then
retrieveStartCommand
${start_command} >> ${KYLIN_HOME}/logs/kylin.out 2>&1 & echo $! > ${KYLIN_HOME}/pid &
+ rm -f $lockfile
echo ""
echo "A new Kylin instance is started by $USER. To stop it, run 'kylin.sh stop'"
@@ -142,6 +151,7 @@ elif [ "$1" == "run" ]
then
retrieveStartCommand
${start_command}
+ rm -f $lockfile
# stop command
elif [ "$1" == "stop" ]
[kylin] 05/14: KYLIN-4226: Skip current unavailable tables when
updating hbase coprocessor
Posted by ni...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
nic pushed a commit to branch 2.6.x
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 9d2007febb7fdfb70917522166b12e5dff09826c
Author: Zhou Kang <zh...@xiaomi.com>
AuthorDate: Wed Oct 30 11:21:35 2019 +0800
KYLIN-4226: Skip current unavailable tables when updating hbase coprocessor
---
.../org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java | 7 +++++++
1 file changed, 7 insertions(+)
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
index 215af1d..51bcbf7 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
@@ -195,7 +195,13 @@ public class DeployCoprocessorCLI {
return tableNames;
}
logger.info("Commit Information: " + commitInfo);
+ int skipTableCnt = 0;
for (String tableName : tableNames) {
+ if (!hbaseAdmin.isTableAvailable(TableName.valueOf(tableName))) {
+ logger.warn("Table: " + tableName + " is not available currently, skip it");
+ skipTableCnt ++;
+ continue;
+ }
HTableDescriptor tableDesc = hbaseAdmin.getTableDescriptor(TableName.valueOf(tableName));
String gitTag = tableDesc.getValue(IRealizationConstants.HTableGitTag);
if (commitInfo.equals(gitTag)) {
@@ -204,6 +210,7 @@ public class DeployCoprocessorCLI {
result.add(tableName);
}
}
+ logger.info("Skip {} tables for not founding in HBase Cluster", skipTableCnt);
logger.info("Filtered tables don't need to deploy coprocessors: " + filteredList);
return result;
}
[kylin] 03/14: KYLIN-4287 segmenPruner satisfy "IN" filter bug
Posted by ni...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
nic pushed a commit to branch 2.6.x
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit b6b2e162a4453799015afd1c529a4539b55c9e8b
Author: zhangbushi5 <84...@qq.com>
AuthorDate: Tue Dec 10 17:15:54 2019 +0800
KYLIN-4287 segmenPruner satisfy "IN" filter bug
---
.../apache/kylin/cube/common/SegmentPruner.java | 8 ++++---
.../kylin/cube/common/SegmentPrunerTest.java | 25 ++++++++++++++++++++++
2 files changed, 30 insertions(+), 3 deletions(-)
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/common/SegmentPruner.java b/core-cube/src/main/java/org/apache/kylin/cube/common/SegmentPruner.java
index ae21a4d..2de62de 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/common/SegmentPruner.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/common/SegmentPruner.java
@@ -137,9 +137,11 @@ public class SegmentPruner {
switch (comp.getOperator()) {
case EQ:
case IN:
- String filterMin = order.min((Set<String>) comp.getValues());
- String filterMax = order.max((Set<String>) comp.getValues());
- return order.compare(filterMin, maxVal) <= 0 && order.compare(minVal, filterMax) <= 0;
+ for (String filterValue : (Set<String>) comp.getValues()) {
+ if (order.compare(filterValue, maxVal) <= 0 && order.compare(minVal, filterValue) <= 0)
+ return true;
+ }
+ return false;
case LT:
return order.compare(minVal, filterVal) < 0;
case LTE:
diff --git a/core-cube/src/test/java/org/apache/kylin/cube/common/SegmentPrunerTest.java b/core-cube/src/test/java/org/apache/kylin/cube/common/SegmentPrunerTest.java
index 69f9b10..5d98d06 100644
--- a/core-cube/src/test/java/org/apache/kylin/cube/common/SegmentPrunerTest.java
+++ b/core-cube/src/test/java/org/apache/kylin/cube/common/SegmentPrunerTest.java
@@ -343,4 +343,29 @@ public class SegmentPrunerTest extends LocalFileMetadataTestCase {
}
}
}
+
+ @Test
+ public void testPruneSegWithFilterIN() {
+ // legacy cube segments does not have DimensionRangeInfo, but with TSRange can do some pruning
+ CubeInstance cube = CubeManager.getInstance(getTestConfig())
+ .getCube("test_kylin_cube_without_slr_left_join_ready_2_segments");
+ TblColRef col = cube.getModel().findColumn("TEST_KYLIN_FACT.CAL_DT");
+ CubeSegment seg = cube.getSegments(SegmentStatusEnum.READY).get(0);
+ TSRange tsRange = seg.getTSRange();
+ String start = DateFormat.formatToTimeStr(tsRange.start.v, "yyyy-MM-dd");
+ CubeSegment seg2 = cube.getSegments(SegmentStatusEnum.READY).get(1);
+ TSRange tsRange2 = seg2.getTSRange();
+ try (SetAndUnsetSystemProp sns = new SetAndUnsetSystemProp("kylin.query.skip-empty-segments", "false")) {
+
+ {
+ TupleFilter inFilter = new ConstantTupleFilter(Sets.newHashSet(start,
+ DateFormat.formatToTimeStr(tsRange2.end.v + 1000 * 60 * 60 * 24L, "yyyy-MM-dd")));
+ TupleFilter filter = compare(col, FilterOperatorEnum.IN, inFilter);
+ SegmentPruner segmentPruner = new SegmentPruner(filter);
+ Assert.assertTrue(segmentPruner.check(seg));
+ Assert.assertFalse(segmentPruner.check(seg2));
+
+ }
+ }
+ }
}