You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by xx...@apache.org on 2023/01/12 09:27:56 UTC
[kylin] branch kylin5 updated (2f6d37c563 -> 92af96e52a)
This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a change to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
from 2f6d37c563 Remove test for RecCandidateTool
new fbee49883d [DIRTY] format code
new a410bd0c08 KYLIN-5389 fix duplicate CC when querying
new a9de3a2a27 KYLIN-5390 Build segment support for overlap segments
new c14359ccd3 [DIRTY] fix partition snapshot build on spark serverless
new 4d3f892b27 KYLIN-5392 slow query can not cancel
new bbcf3df8d7 KYLIN-5393 fix bug when double quote in pushdown sql
new 8ada2027f4 [DIRTY] fix favorite rules return null
new 4bb90cbab3 KYLIN-5394 Project-level push-down query does not take effect (jdbc push-down query)
new ae22f91de9 KYLIN-5395 sync column comment to dimension name and measure comment (#29499)
new 092bbed478 KYLIN-5353 add UT
new a3a709853c KYLIN-5396 fix the version of pom in Kylin-it
new 6fbd7b103c [DIRTY] check multi partition column in load data
new 4099e3b508 [DIRTY] upgrade spark version
new 60ad2aa86d KYLIN-5397 Support sum_lc function
new d3ecc5a084 KYLIN-5398 Flame graph availability fix
new 1a00e9ac57 KYLIN-5399 refactor core-common
new 92af96e52a [DIRTY] upgrade spark version
The 17 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../libasyncProfiler-linux-arm64.so | Bin 0 -> 298256 bytes
.../libasyncProfiler-linux-x64.so | Bin 0 -> 303903 bytes
build/async-profiler-lib/libasyncProfiler-mac.so | Bin 0 -> 634272 bytes
build/release/compress.sh | 5 +
pom.xml | 39 ++-
src/assembly/pom.xml | 4 +
src/common-service/pom.xml | 5 +-
.../apache/kylin/rest/config/MetricsConfig.java | 4 +-
.../kylin/rest/controller/NBasicController.java | 17 +-
.../interceptor/ReloadAuthoritiesInterceptor.java | 5 +-
.../rest/security/NUnauthorisedEntryPoint.java | 3 +-
.../apache/kylin/rest/service/ProjectService.java | 2 +-
.../org/apache/kylin/rest/util/SparkUIUtil.java | 3 +-
.../kylin/rest/service/AccessServiceTest.java | 9 +-
.../kylin/rest/service/AclTCRServiceTest.java | 4 +-
.../task/QueryHistoryTaskSchedulerRunnerTest.java | 2 +-
.../task/QueryHistoryTaskSchedulerTest.java | 6 +-
src/core-common/pom.xml | 222 +++------------
.../java/org/apache/kylin/common/KapConfig.java | 13 +-
.../org/apache/kylin/common/KylinConfigBase.java | 19 +-
.../AsyncProfilerConstants.java} | 23 +-
.../java/org/apache/kylin/common/msg/Message.java | 2 +-
.../common/persistence/metadata/jdbc/JdbcUtil.java | 2 +-
.../common/persistence/transaction/UnitOfWork.java | 22 +-
.../org/apache/kylin/common/util/DateFormat.java | 29 +-
.../java/org/apache/kylin/common/util/Unsafe.java | 8 +-
.../libasyncProfiler-linux-arm64.so | Bin 0 -> 298256 bytes
.../libasyncProfiler-linux-x64.so | Bin 0 -> 303903 bytes
.../async-profiler-lib/libasyncProfiler-mac.so | Bin 0 -> 634272 bytes
.../resources/kylin_error_msg_conf_cn.properties | 2 +-
.../resources/kylin_error_msg_conf_en.properties | 2 +-
.../apache/kylin/common/KylinConfigBaseTest.java | 10 +
.../apache/kylin/common/util/DateFormatTest.java | 51 ++++
src/core-job/pom.xml | 16 ++
.../kylin/common/util/SystemInfoCollector.java | 0
.../org/apache/kylin/job/common/SegmentUtil.java | 18 +-
.../apache/kylin/metrics/HdfsCapacityMetrics.java | 0
.../org/apache/kylin/job/common/SegmentsTest.java | 53 +++-
.../kylin/metrics/HdfsCapacityMetricsTest.java | 0
src/core-metadata/pom.xml | 26 +-
.../apache/kylin/measure/MeasureTypeFactory.java | 2 +
.../apache/kylin/measure/sumlc/SumLCCounter.java | 96 +++++++
.../kylin/measure/sumlc/SumLCMeasureType.java | 172 ++++++++++++
.../apache/kylin/metadata/datatype/DataType.java | 10 +-
.../apache/kylin/metadata/model/ColumnDesc.java | 8 +
.../kylin/metadata/model/ComputedColumnDesc.java | 9 -
.../apache/kylin/metadata/model/FunctionDesc.java | 69 +++--
.../org/apache/kylin/metadata/model/TableDesc.java | 11 +
.../org/apache/kylin/metadata/model/TblColRef.java | 4 +-
.../metadata/model/schema/ModelEdgeCollector.java | 12 +-
.../kylin/metadata/project/NProjectManager.java | 6 +
.../kylin/source/adhocquery/IPushDownRunner.java | 7 +
.../org/apache/kylin/source/jdbc/H2Database.java | 2 +-
.../metadata/model/util/FunctionDescTest.java | 12 +-
.../metadata/project/NProjectManagerTest.java | 5 +-
src/{core-job => core-metrics}/pom.xml | 128 +++++----
.../kylin/common/metrics/MetricsCategory.java | 0
.../apache/kylin/common/metrics/MetricsConfig.java | 0
.../kylin/common/metrics/MetricsController.java | 0
.../apache/kylin/common/metrics/MetricsGroup.java | 0
.../common/metrics/MetricsInfluxdbReporter.java | 0
.../apache/kylin/common/metrics/MetricsName.java | 0
.../apache/kylin/common/metrics/MetricsObject.java | 0
.../kylin/common/metrics/MetricsObjectType.java | 0
.../kylin/common/metrics/MetricsReporter.java | 0
.../apache/kylin/common/metrics/MetricsTag.java | 0
.../common/metrics/context/ClusterContext.java | 0
.../kylin/common/metrics/context/ClusterInfo.java | 0
.../common/metrics/gauges/QueryRatioGauge.java | 0
.../metrics/prometheus/PrometheusMetrics.java | 0
.../common/metrics/reporter/InfluxdbReporter.java | 2 +-
.../metrics/reporter/ServerModeMetricFilter.java | 0
.../common/metrics/service/InfluxDBInstance.java | 4 +-
.../metrics/service/JobStatusMonitorMetric.java | 0
.../kylin/common/metrics/service/MonitorDao.java | 0
.../common/metrics/service/MonitorMetric.java | 0
.../metrics/service/MonitorMetricOperation.java | 0
.../common/metrics/service/QueryMonitorMetric.java | 0
.../apache/kylin/common/util/InfluxDBUtils.java | 0
.../kylin/common/metric/InfluxDBInstanceTest.java | 0
.../kylin/common/metric/InfluxMetricsTest.java | 0
.../kylin/common/metric/MetricsControllerTest.java | 16 +-
.../apache/kylin/common/metric/MonitorDaoTest.java | 0
.../kylin/rest/controller/BaseController.java | 19 +-
.../kylin/rest/controller/SegmentController.java | 3 +-
.../kylin/rest/filter/SegmentsRequestFilter.java | 3 +-
.../org/apache/kylin/rest/service/JobService.java | 3 +-
.../kylin/rest/service/ModelBuildService.java | 18 +-
.../kylin/rest/service/ModelServiceBuildTest.java | 4 +-
src/datasource-sdk/pom.xml | 27 +-
.../sdk/datasource/PushDownRunnerSDKImpl.java | 7 +-
.../sdk/datasource/PushdownRunnerSDKImplTest.java | 6 +-
src/datasource-service/pom.xml | 4 +
src/{core-job => distributed-lock-ext}/pom.xml | 107 ++++----
.../lock/curator/CuratorDistributedLock.java | 0
.../curator/CuratorDistributedLockFactory.java | 0
.../lock/jdbc/JdbcDistributedLockFactory.java | 0
.../common/lock/jdbc/JdbcDistributedLockUtil.java | 24 +-
.../java/org/apache/kylin/common/util/ZKUtil.java | 8 +-
.../common/lock/DistributedLockFactoryTest.java | 11 +-
.../curator/CuratorDistributedLockFactoryTest.java | 43 +--
.../lock/jdbc/JdbcDistributedLockFactoryTest.java | 13 +-
.../localmeta/lib/libasyncProfiler-linux-arm64.so | Bin 0 -> 298256 bytes
.../localmeta/lib/libasyncProfiler-linux-x64.so | Bin 0 -> 303903 bytes
.../localmeta/lib/libasyncProfiler-mac.so | Bin 0 -> 634272 bytes
.../localmeta/metadata/_global/project/sum_lc.json | 35 +++
.../f35f2937-9e4d-347a-7465-d64df939e7d6.json | 13 +
.../f35f2937-9e4d-347a-7465-d64df939e7d6.json | 43 +++
.../f35f2937-9e4d-347a-7465-d64df939e7d6.json | 267 ++++++++++++++++++
.../metadata/sum_lc/table/SSB.SUMLC_EXTEND_4X.json | 78 ++++++
src/kylin-it/pom.xml | 4 +-
.../kylin/newten/NBadQueryAndPushDownTest.java | 5 +-
.../apache/kylin/newten/SlowQueryDetectorTest.java | 13 +-
.../java/org/apache/kylin/query/KylinTestBase.java | 4 +-
.../kylin/query/rules/CalciteRuleTestBase.java | 6 +-
.../kylin/streaming/StreamingMergeEntryTest.java | 9 +
.../java/org/apache/kylin/util/ExecAndComp.java | 16 +-
.../rest/controller/open/OpenModelController.java | 17 ++
.../controller/open/OpenModelControllerTest.java | 16 ++
.../kylin/common/util/AddTableNameSqlVisitor.java | 0
.../common/util/ModifyTableNameSqlVisitor.java | 0
.../response/SynchronizedCommentsResponse.java | 220 +++++++++++++++
.../kylin/rest/service/ModelSemanticHelper.java | 15 +-
.../apache/kylin/rest/service/ModelService.java | 76 +++---
.../rest/service/ModelServiceBrokenRepairTest.java | 15 ++
.../kylin/rest/service/ModelServiceTest.java | 179 ++++++++++++-
.../kylin/rest/service/ProjectServiceTest.java | 8 +-
.../3580693c-d1fd-7697-8ee8-0474f81c413d.json | 13 +
.../722b027b-8906-379b-cf4f-ac2055ee528b.json | 2 +-
.../8dc4c289-c95d-0402-31e4-b123f9ef553f.json | 2 +-
...n => 3580693c-d1fd-7697-8ee8-0474f81c413d.json} | 8 +-
.../722b027b-8906-379b-cf4f-ac2055ee528b.json | 1 +
.../8dc4c289-c95d-0402-31e4-b123f9ef553f.json | 2 +-
.../3580693c-d1fd-7697-8ee8-0474f81c413d.json} | 172 ++++++------
.../722b027b-8906-379b-cf4f-ac2055ee528b.json | 2 +-
.../8dc4c289-c95d-0402-31e4-b123f9ef553f.json | 2 +-
.../metadata/ssb/table/SSB.CUSTOMER.json | 2 +-
.../metadata/ssb/table/SSB.LINEORDER.json | 2 +-
.../metadata/ssb/table/SSB.SUPPLIER.json | 2 +-
.../model_request_broken_aftr_reload.json | 2 +-
.../model_request_broken_aftr_reload_2.json | 2 +-
...2.json => model_request_with_timestamp_cc.json} | 172 ++++++------
.../org/apache/kylin/query/IQueryTransformer.java} | 12 +-
.../org/apache/kylin/query/SlowQueryDetector.java | 2 +-
.../kylin/query/relnode/KapAggregateRel.java | 21 +-
.../kylin/query/relnode/OLAPAggregateRel.java | 2 +-
.../org/apache/kylin/query/schema/OLAPTable.java | 2 +-
.../security/HackSelectStarWithColumnACL.java | 12 +-
.../org/apache/kylin/query/security/RowFilter.java | 6 +-
.../kylin/query/util/ConvertToComputedColumn.java | 24 +-
.../query/util/DateNumberFilterTransformer.java | 11 +-
.../kylin/query/util/DefaultQueryTransformer.java | 3 +-
.../apache/kylin/query/util/EscapeTransformer.java | 6 +-
.../kylin/query/util/KeywordDefaultDirtyHack.java | 3 +-
.../apache/kylin/query/util/PowerBIConverter.java | 6 +-
.../org/apache/kylin/query/util/PushDownUtil.java | 12 +-
.../apache/kylin/query/util/QueryAliasMatcher.java | 71 ++---
.../util/{KapQueryUtil.java => QueryUtil.java} | 297 ++++++++++++++-------
.../query/util/RestoreFromComputedColumn.java | 74 +++--
.../query/util/WithToSubQueryTransformer.java | 6 +-
.../rest/controller/NQueryControllerTest.java | 75 ++----
.../SnapshotSourceTableStatsServiceTest.scala | 7 +-
.../kylin/rest/service/QueryCacheManager.java | 5 +-
.../apache/kylin/rest/service/QueryService.java | 120 ++++-----
.../org/apache/kylin/rest/util/QueryUtils.java | 17 +-
.../rest/metrics/QueryMetricsContextTest.java | 15 +-
.../kylin/rest/service/ModelServiceQueryTest.java | 8 +-
.../rest/service/QueryHistoryServiceTest.java | 2 +-
.../kylin/rest/service/QueryServiceTest.java | 36 +--
src/query/pom.xml | 4 +
.../optrule/AbstractAggCaseWhenFunctionRule.java | 4 +-
.../optrule/CountDistinctCaseWhenFunctionRule.java | 6 +-
.../query/optrule/KapAggFilterTransposeRule.java | 4 +-
.../kap/query/optrule/KapAggJoinTransposeRule.java | 4 +-
.../kap/query/optrule/KapAggProjectMergeRule.java | 4 +-
.../query/optrule/KapAggProjectTransposeRule.java | 4 +-
.../query/optrule/KapCountDistinctJoinRule.java | 4 +-
.../kap/query/optrule/KapSumCastTransposeRule.java | 4 +-
.../kylin/query/engine/QueryRoutingEngine.java | 29 +-
.../org/apache/kylin/query/util/QueryHelper.java | 15 +-
.../org/apache/kylin/query/util/QueryUtil.java | 171 ------------
.../kylin/query/engine/AsyncQueryJobTest.java | 6 +-
.../kylin/query/engine/QueryRoutingEngineTest.java | 16 ++
.../apache/kylin/query/util/KapQueryUtilTest.java | 60 -----
.../org/apache/kylin/query/util/QueryUtilTest.java | 201 +++++++++-----
.../kap/secondstorage/tdvt/TDVTHiveTest.java | 9 +-
.../acfde546-2cc9-4eec-bc92-e3bd46d4e2ee.json | 6 +
.../acfde546-2cc9-4eec-bc92-e3bd46d4e2ee.json | 6 +
.../kyligence/kap/clickhouse/job/DataLoader.java | 6 +-
.../service/ModelServiceWithSecondStorageTest.java | 4 +-
src/server/pom.xml | 24 ++
.../org/apache/kylin/rest/QueryNodeFilter.java | 7 +-
src/server/src/main/resources/application.yaml | 2 +-
src/spark-project/engine-build-sdk/pom.xml | 1 -
src/spark-project/engine-spark/pom.xml | 4 +
.../kylin/engine/spark/job/NSparkExecutable.java | 55 ++--
.../spark/merger/AfterBuildResourceMerger.java | 6 +-
.../engine/spark/builder/SegmentFlatTable.scala | 35 ++-
.../engine/spark/builder/SnapshotBuilder.scala | 31 ++-
.../kylin/engine/spark/job/CuboidAggregator.scala | 41 +--
.../kylin/engine/spark/job/FlatTableHelper.scala | 18 +-
.../job/stage/build/FlatTableAndDictBase.scala | 12 +-
.../kylin/engine/spark/job/JobManagerTest.java | 29 +-
.../kylin/query/pushdown/DataSourceConfig.java} | 32 ++-
.../pushdown/JdbcPushDownConnectionManager.java | 26 +-
.../query/pushdown/PushDownRunnerJdbcImpl.java | 11 +-
.../query/pushdown/PushDownRunnerSparkImpl.java | 7 +-
.../kylin/query/asyncprofiler/AsyncProfiling.scala | 9 +-
.../QueryAsyncProfilerDriverPlugin.scala | 7 +-
.../kylin/query/pushdown/SparkSqlClient.scala | 53 ++--
.../kylin/query/runtime/plan/AggregatePlan.scala | 11 +-
.../kylin/query/runtime/plan/ResultPlan.scala | 47 ++--
.../scala/org/apache/spark/sql/KapFunctions.scala | 14 +-
.../sql/catalyst/expressions/ExpressionUtils.scala | 31 ++-
.../sql/catalyst/expressions/KapExpresssions.scala | 64 ++++-
.../org/apache/spark/sql/udf/SparderAggFun.scala | 11 +-
.../query/pushdown/PushDownRunnerJdbcImplTest.java | 61 ++++-
.../pushdown/PushDownRunnerSparkImplTest.java | 12 +-
.../query/asyncprofiler/AsyncPluginWithMeta.scala | 8 +-
.../query/asyncprofiler/AsyncProfilingTest.scala | 45 ++--
.../QueryAsyncProfilerDriverPluginTest.scala | 27 +-
src/spark-project/spark-common/pom.xml | 5 +
.../kylin/common/asyncprofiler/AsyncArchUtil.java | 77 ++++++
.../kylin/common/asyncprofiler/AsyncProfiler.java | 81 +++---
.../common/asyncprofiler/AsyncProfilerUtils.java | 4 +
.../org/apache/kylin/source/dfs/DFSFileTable.java | 0
.../kylin/source/dfs/DFSFileTableReader.java | 4 +-
.../libasyncProfiler-linux-arm64.so | Bin 0 -> 298256 bytes
.../libasyncProfiler-linux-x64.so | Bin 0 -> 303903 bytes
.../async-profiler-lib/libasyncProfiler-mac.so | Bin 0 -> 634272 bytes
.../async-profiler-lib/linux64/libasyncProfiler.so | Bin
.../async-profiler-lib/macOS/libasyncProfiler.so | Bin
.../AsyncProfilerExecutorPlugin.scala | 8 +-
.../common/asyncprofiler/AsyncProfilerTool.scala | 10 +-
.../kylin/common/asyncprofiler/Message.scala | 4 +-
.../BuildAsyncProfilerDriverPlugin.scala | 9 +-
.../apache/spark/sql/LayoutEntityConverter.scala | 6 +-
.../spark/sql/udaf/NullSafeValueSerializer.scala | 30 ++-
.../scala/org/apache/spark/sql/udaf/SumLC.scala | 191 +++++++++++++
.../common/asyncprofiler/AsyncArchUtilTest.java} | 31 +--
.../common/asyncprofiler/AsyncProfilerTest.java | 33 +--
.../asyncprofiler/AsyncProfilerToolTest.java | 24 ++
.../asyncprofiler/AsyncProfilerUtilsTest.java | 22 +-
.../kylin/common/asyncprofiler/MessageTest.java | 3 +-
.../org/apache/kylin/common/CompareSupport.scala | 4 +-
.../org/apache/kylin/common/QuerySupport.scala | 11 +-
.../scala/org/apache/kylin/common/SSSource.scala | 8 +-
.../org/apache/kylin/common/YarnSupport.scala | 1 -
src/systools/pom.xml | 22 +-
249 files changed, 3742 insertions(+), 1935 deletions(-)
create mode 100644 build/async-profiler-lib/libasyncProfiler-linux-arm64.so
create mode 100644 build/async-profiler-lib/libasyncProfiler-linux-x64.so
create mode 100644 build/async-profiler-lib/libasyncProfiler-mac.so
copy src/core-common/src/main/java/org/apache/kylin/common/{metrics/MetricsObjectType.java => constant/AsyncProfilerConstants.java} (66%)
create mode 100644 src/core-common/src/main/resources/async-profiler-lib/libasyncProfiler-linux-arm64.so
create mode 100644 src/core-common/src/main/resources/async-profiler-lib/libasyncProfiler-linux-x64.so
create mode 100644 src/core-common/src/main/resources/async-profiler-lib/libasyncProfiler-mac.so
rename src/{core-common => core-job}/src/main/java/org/apache/kylin/common/util/SystemInfoCollector.java (100%)
rename src/{core-metadata => core-job}/src/main/java/org/apache/kylin/metrics/HdfsCapacityMetrics.java (100%)
rename src/{core-metadata => core-job}/src/test/java/org/apache/kylin/metrics/HdfsCapacityMetricsTest.java (100%)
create mode 100644 src/core-metadata/src/main/java/org/apache/kylin/measure/sumlc/SumLCCounter.java
create mode 100644 src/core-metadata/src/main/java/org/apache/kylin/measure/sumlc/SumLCMeasureType.java
rename src/{source-hive => core-metadata}/src/main/java/org/apache/kylin/source/jdbc/H2Database.java (100%)
copy src/{core-job => core-metrics}/pom.xml (51%)
copy src/{core-common => core-metrics}/src/main/java/org/apache/kylin/common/metrics/MetricsCategory.java (100%)
rename src/{core-common => core-metrics}/src/main/java/org/apache/kylin/common/metrics/MetricsConfig.java (100%)
rename src/{core-common => core-metrics}/src/main/java/org/apache/kylin/common/metrics/MetricsController.java (100%)
rename src/{core-common => core-metrics}/src/main/java/org/apache/kylin/common/metrics/MetricsGroup.java (100%)
rename src/{core-common => core-metrics}/src/main/java/org/apache/kylin/common/metrics/MetricsInfluxdbReporter.java (100%)
rename src/{core-common => core-metrics}/src/main/java/org/apache/kylin/common/metrics/MetricsName.java (100%)
rename src/{core-common => core-metrics}/src/main/java/org/apache/kylin/common/metrics/MetricsObject.java (100%)
copy src/{core-common => core-metrics}/src/main/java/org/apache/kylin/common/metrics/MetricsObjectType.java (100%)
rename src/{core-common => core-metrics}/src/main/java/org/apache/kylin/common/metrics/MetricsReporter.java (100%)
rename src/{core-common => core-metrics}/src/main/java/org/apache/kylin/common/metrics/MetricsTag.java (100%)
rename src/{core-common => core-metrics}/src/main/java/org/apache/kylin/common/metrics/context/ClusterContext.java (100%)
rename src/{core-common => core-metrics}/src/main/java/org/apache/kylin/common/metrics/context/ClusterInfo.java (100%)
rename src/{core-common => core-metrics}/src/main/java/org/apache/kylin/common/metrics/gauges/QueryRatioGauge.java (100%)
rename src/{core-common => core-metrics}/src/main/java/org/apache/kylin/common/metrics/prometheus/PrometheusMetrics.java (100%)
rename src/{core-common => core-metrics}/src/main/java/org/apache/kylin/common/metrics/reporter/InfluxdbReporter.java (100%)
rename src/{core-common => core-metrics}/src/main/java/org/apache/kylin/common/metrics/reporter/ServerModeMetricFilter.java (100%)
rename src/{core-common => core-metrics}/src/main/java/org/apache/kylin/common/metrics/service/InfluxDBInstance.java (100%)
rename src/{core-common => core-metrics}/src/main/java/org/apache/kylin/common/metrics/service/JobStatusMonitorMetric.java (100%)
rename src/{core-common => core-metrics}/src/main/java/org/apache/kylin/common/metrics/service/MonitorDao.java (100%)
rename src/{core-common => core-metrics}/src/main/java/org/apache/kylin/common/metrics/service/MonitorMetric.java (100%)
copy src/{core-common => core-metrics}/src/main/java/org/apache/kylin/common/metrics/service/MonitorMetricOperation.java (100%)
rename src/{core-common => core-metrics}/src/main/java/org/apache/kylin/common/metrics/service/QueryMonitorMetric.java (100%)
rename src/{core-common => core-metrics}/src/main/java/org/apache/kylin/common/util/InfluxDBUtils.java (100%)
rename src/{core-common => core-metrics}/src/test/java/org/apache/kylin/common/metric/InfluxDBInstanceTest.java (100%)
rename src/{core-common => core-metrics}/src/test/java/org/apache/kylin/common/metric/InfluxMetricsTest.java (100%)
rename src/{core-common => core-metrics}/src/test/java/org/apache/kylin/common/metric/MetricsControllerTest.java (89%)
rename src/{core-common => core-metrics}/src/test/java/org/apache/kylin/common/metric/MonitorDaoTest.java (100%)
copy src/{core-job => distributed-lock-ext}/pom.xml (51%)
rename src/{core-common => distributed-lock-ext}/src/main/java/org/apache/kylin/common/lock/curator/CuratorDistributedLock.java (100%)
rename src/{core-common => distributed-lock-ext}/src/main/java/org/apache/kylin/common/lock/curator/CuratorDistributedLockFactory.java (100%)
rename src/{core-common => distributed-lock-ext}/src/main/java/org/apache/kylin/common/lock/jdbc/JdbcDistributedLockFactory.java (100%)
rename src/{core-common => distributed-lock-ext}/src/main/java/org/apache/kylin/common/lock/jdbc/JdbcDistributedLockUtil.java (99%)
rename src/{core-common => distributed-lock-ext}/src/main/java/org/apache/kylin/common/util/ZKUtil.java (96%)
rename src/{core-common => distributed-lock-ext}/src/test/java/org/apache/kylin/common/lock/DistributedLockFactoryTest.java (98%)
rename src/{core-common => distributed-lock-ext}/src/test/java/org/apache/kylin/common/lock/curator/CuratorDistributedLockFactoryTest.java (73%)
rename src/{core-common => distributed-lock-ext}/src/test/java/org/apache/kylin/common/lock/jdbc/JdbcDistributedLockFactoryTest.java (91%)
create mode 100644 src/examples/test_case_data/localmeta/lib/libasyncProfiler-linux-arm64.so
create mode 100644 src/examples/test_case_data/localmeta/lib/libasyncProfiler-linux-x64.so
create mode 100644 src/examples/test_case_data/localmeta/lib/libasyncProfiler-mac.so
create mode 100644 src/examples/test_case_data/localmeta/metadata/_global/project/sum_lc.json
create mode 100644 src/examples/test_case_data/localmeta/metadata/sum_lc/dataflow/f35f2937-9e4d-347a-7465-d64df939e7d6.json
create mode 100644 src/examples/test_case_data/localmeta/metadata/sum_lc/index_plan/f35f2937-9e4d-347a-7465-d64df939e7d6.json
create mode 100644 src/examples/test_case_data/localmeta/metadata/sum_lc/model_desc/f35f2937-9e4d-347a-7465-d64df939e7d6.json
create mode 100644 src/examples/test_case_data/localmeta/metadata/sum_lc/table/SSB.SUMLC_EXTEND_4X.json
rename src/{core-common => modeling-service}/src/main/java/org/apache/kylin/common/util/AddTableNameSqlVisitor.java (100%)
rename src/{core-common => modeling-service}/src/main/java/org/apache/kylin/common/util/ModifyTableNameSqlVisitor.java (100%)
create mode 100644 src/modeling-service/src/main/java/org/apache/kylin/rest/response/SynchronizedCommentsResponse.java
create mode 100644 src/modeling-service/src/test/resources/ut_meta/broken_repair_test/metadata/ssb/dataflow/3580693c-d1fd-7697-8ee8-0474f81c413d.json
copy src/modeling-service/src/test/resources/ut_meta/broken_repair_test/metadata/ssb/index_plan/{8dc4c289-c95d-0402-31e4-b123f9ef553f.json => 3580693c-d1fd-7697-8ee8-0474f81c413d.json} (79%)
copy src/modeling-service/src/test/resources/ut_meta/broken_repair_test/{model_request/model_request_broken_aftr_reload_2.json => metadata/ssb/model_desc/3580693c-d1fd-7697-8ee8-0474f81c413d.json} (61%)
copy src/modeling-service/src/test/resources/ut_meta/broken_repair_test/model_request/{model_request_broken_aftr_reload_2.json => model_request_with_timestamp_cc.json} (61%)
rename src/{core-common/src/main/java/org/apache/kylin/common/metrics/service/MonitorMetricOperation.java => query-common/src/main/java/org/apache/kylin/query/IQueryTransformer.java} (79%)
rename src/query-common/src/main/java/org/apache/kylin/query/util/{KapQueryUtil.java => QueryUtil.java} (65%)
delete mode 100644 src/query/src/main/java/org/apache/kylin/query/util/QueryUtil.java
delete mode 100644 src/query/src/test/java/org/apache/kylin/query/util/KapQueryUtilTest.java
rename src/{core-common/src/main/java/org/apache/kylin/common/metrics/MetricsCategory.java => spark-project/sparder/src/main/java/org/apache/kylin/query/pushdown/DataSourceConfig.java} (53%)
create mode 100644 src/spark-project/spark-common/src/main/java/org/apache/kylin/common/asyncprofiler/AsyncArchUtil.java
rename src/{core-common => spark-project/spark-common}/src/main/java/org/apache/kylin/common/asyncprofiler/AsyncProfiler.java (52%)
rename src/{core-common => spark-project/spark-common}/src/main/java/org/apache/kylin/common/asyncprofiler/AsyncProfilerUtils.java (97%)
rename src/{core-metadata => spark-project/spark-common}/src/main/java/org/apache/kylin/source/dfs/DFSFileTable.java (100%)
rename src/{core-metadata => spark-project/spark-common}/src/main/java/org/apache/kylin/source/dfs/DFSFileTableReader.java (98%)
create mode 100644 src/spark-project/spark-common/src/main/resources/async-profiler-lib/libasyncProfiler-linux-arm64.so
create mode 100644 src/spark-project/spark-common/src/main/resources/async-profiler-lib/libasyncProfiler-linux-x64.so
create mode 100644 src/spark-project/spark-common/src/main/resources/async-profiler-lib/libasyncProfiler-mac.so
rename src/{core-common => spark-project/spark-common}/src/main/resources/async-profiler-lib/linux64/libasyncProfiler.so (100%)
rename src/{core-common => spark-project/spark-common}/src/main/resources/async-profiler-lib/macOS/libasyncProfiler.so (100%)
rename src/{core-common => spark-project/spark-common}/src/main/scala/org/apache/kylin/common/asyncprofiler/AsyncProfilerExecutorPlugin.scala (90%)
rename src/{core-common => spark-project/spark-common}/src/main/scala/org/apache/kylin/common/asyncprofiler/AsyncProfilerTool.scala (86%)
rename src/{core-common => spark-project/spark-common}/src/main/scala/org/apache/kylin/common/asyncprofiler/Message.scala (98%)
create mode 100644 src/spark-project/spark-common/src/main/scala/org/apache/spark/sql/udaf/SumLC.scala
rename src/{core-common/src/main/java/org/apache/kylin/common/metrics/MetricsObjectType.java => spark-project/spark-common/src/test/java/org/apache/kylin/common/asyncprofiler/AsyncArchUtilTest.java} (57%)
rename src/{core-common => spark-project/spark-common}/src/test/java/org/apache/kylin/common/asyncprofiler/AsyncProfilerTest.java (72%)
rename src/{core-common => spark-project/spark-common}/src/test/java/org/apache/kylin/common/asyncprofiler/AsyncProfilerToolTest.java (80%)
rename src/{core-common => spark-project/spark-common}/src/test/java/org/apache/kylin/common/asyncprofiler/AsyncProfilerUtilsTest.java (83%)
rename src/{core-common => spark-project/spark-common}/src/test/java/org/apache/kylin/common/asyncprofiler/MessageTest.java (96%)
[kylin] 04/17: [DIRTY] fix partition snapshot build on spark serverless
Posted by xx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit c14359ccd3b55704f4168b70cc9e8df89732b751
Author: xingjian <xi...@kyligence.io>
AuthorDate: Thu Nov 10 10:08:27 2022 +0800
[DIRTY] fix partition snapshot build on spark serverless
---
.../engine/spark/builder/SnapshotBuilder.scala | 31 ++++++++++++----------
1 file changed, 17 insertions(+), 14 deletions(-)
diff --git a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/builder/SnapshotBuilder.scala b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/builder/SnapshotBuilder.scala
index ce5a080601..b077112498 100644
--- a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/builder/SnapshotBuilder.scala
+++ b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/builder/SnapshotBuilder.scala
@@ -18,29 +18,28 @@
package org.apache.kylin.engine.spark.builder
-import java.io.IOException
-import java.util
-import java.util.concurrent.{ConcurrentHashMap, ConcurrentMap, Executors}
-import java.util.{Objects, UUID}
import com.google.common.collect.Maps
-import org.apache.kylin.engine.spark.NSparkCubingEngine
-import org.apache.kylin.engine.spark.job.{DFChooser, KylinBuildEnv}
-import org.apache.kylin.engine.spark.utils.{FileNames, LogUtils}
-import org.apache.kylin.metadata.model.{NDataModel, NTableMetadataManager}
-import org.apache.kylin.metadata.project.NProjectManager
import org.apache.commons.codec.digest.DigestUtils
import org.apache.hadoop.fs.{FileStatus, FileSystem, Path, PathFilter}
import org.apache.kylin.common.KylinConfig.SetAndUnsetThreadLocalConfig
import org.apache.kylin.common.persistence.transaction.UnitOfWork
-import org.apache.kylin.common.{KapConfig, KylinConfig}
import org.apache.kylin.common.util.HadoopUtil
-import org.apache.kylin.metadata.model.{TableDesc, TableExtDesc}
+import org.apache.kylin.common.{KapConfig, KylinConfig}
+import org.apache.kylin.engine.spark.NSparkCubingEngine
+import org.apache.kylin.engine.spark.job.{DFChooser, KylinBuildEnv}
+import org.apache.kylin.engine.spark.utils.{FileNames, LogUtils}
+import org.apache.kylin.metadata.model.{NDataModel, NTableMetadataManager, TableDesc, TableExtDesc}
+import org.apache.kylin.metadata.project.NProjectManager
import org.apache.kylin.source.SourceFactory
import org.apache.spark.internal.Logging
import org.apache.spark.sql.hive.utils.ResourceDetectUtils
-import org.apache.spark.sql.{Dataset, Encoders, Row, SparderEnv, SparkSession}
+import org.apache.spark.sql._
import org.apache.spark.utils.ProxyThreadUtils
+import java.io.IOException
+import java.util
+import java.util.concurrent.{ConcurrentHashMap, ConcurrentMap, Executors}
+import java.util.{Objects, UUID}
import scala.collection.JavaConverters._
import scala.concurrent.duration._
import scala.concurrent.{ExecutionContext, Future}
@@ -384,7 +383,7 @@ class SnapshotBuilder(var jobId: String) extends Logging with Serializable {
val resourcePath = baseDir + "/" + snapshotTablePath
var hadoopConf = SparderEnv.getHadoopConfiguration()
if (kylinConfig.getClusterManagerClassName.contains("AWSServerless")) {
- hadoopConf = ss.sparkContext.hadoopConfiguration
+ hadoopConf = ss.sparkContext.hadoopConfiguration
}
val (repartitionNum, sizeMB) = try {
val sizeInMB = ResourceDetectUtils.getPaths(sourceData.queryExecution.sparkPlan)
@@ -464,8 +463,12 @@ class SnapshotBuilder(var jobId: String) extends Logging with Serializable {
hadoopConf = sourceData.sparkSession.sparkContext.hadoopConfiguration
}
try {
+ var hadoopConf = SparderEnv.getHadoopConfiguration()
+ if (kylinConfig.getClusterManagerClassName.contains("AWSServerless")) {
+ hadoopConf = sourceData.sparkSession.sparkContext.hadoopConfiguration
+ }
val sizeInMB = ResourceDetectUtils.getPaths(sourceData.queryExecution.sparkPlan)
- .map(path => HadoopUtil.getContentSummary(path.getFileSystem(SparderEnv.getHadoopConfiguration()), path).getLength)
+ .map(path => HadoopUtil.getContentSummary(path.getFileSystem(hadoopConf), path).getLength)
.sum * 1.0 / MB
val num = Math.ceil(sizeInMB / KylinBuildEnv.get().kylinConfig.getSnapshotShardSizeMB).intValue()
(num, sizeInMB)
[kylin] 12/17: [DIRTY] check multi partition column in load data
Posted by xx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 6fbd7b103cf89908a66d8b2e8a725df91303b679
Author: Zhixiong Chen <ch...@apache.org>
AuthorDate: Thu Nov 3 14:21:08 2022 +0800
[DIRTY] check multi partition column in load data
Co-authored-by: chenzhx <ch...@apache.io>
---
.../model_desc/acfde546-2cc9-4eec-bc92-e3bd46d4e2ee.json | 6 ++++++
.../model_desc/acfde546-2cc9-4eec-bc92-e3bd46d4e2ee.json | 6 ++++++
.../src/main/java/io/kyligence/kap/clickhouse/job/DataLoader.java | 6 ++++--
3 files changed, 16 insertions(+), 2 deletions(-)
diff --git a/src/second-storage/clickhouse-it/src/test/resources/ut_meta/metadata/table_index_incremental/model_desc/acfde546-2cc9-4eec-bc92-e3bd46d4e2ee.json b/src/second-storage/clickhouse-it/src/test/resources/ut_meta/metadata/table_index_incremental/model_desc/acfde546-2cc9-4eec-bc92-e3bd46d4e2ee.json
index ddb2af4e3e..29230d6c3c 100644
--- a/src/second-storage/clickhouse-it/src/test/resources/ut_meta/metadata/table_index_incremental/model_desc/acfde546-2cc9-4eec-bc92-e3bd46d4e2ee.json
+++ b/src/second-storage/clickhouse-it/src/test/resources/ut_meta/metadata/table_index_incremental/model_desc/acfde546-2cc9-4eec-bc92-e3bd46d4e2ee.json
@@ -5,6 +5,12 @@
"owner": "ADMIN",
"join_tables": [],
"all_named_columns": [
+ {
+ "id": 1,
+ "name": "TEST_KYLIN_FACT_CAL_DT",
+ "column": "TEST_KYLIN_FACT.CAL_DT",
+ "status": "TOMB"
+ },
{
"id": 2,
"name": "TEST_KYLIN_FACT_ORDER_ID",
diff --git a/src/second-storage/clickhouse-it/src/test/resources/ut_meta/metadata/table_index_incremental_with_int_date/model_desc/acfde546-2cc9-4eec-bc92-e3bd46d4e2ee.json b/src/second-storage/clickhouse-it/src/test/resources/ut_meta/metadata/table_index_incremental_with_int_date/model_desc/acfde546-2cc9-4eec-bc92-e3bd46d4e2ee.json
index c5c59ff945..613e26508a 100644
--- a/src/second-storage/clickhouse-it/src/test/resources/ut_meta/metadata/table_index_incremental_with_int_date/model_desc/acfde546-2cc9-4eec-bc92-e3bd46d4e2ee.json
+++ b/src/second-storage/clickhouse-it/src/test/resources/ut_meta/metadata/table_index_incremental_with_int_date/model_desc/acfde546-2cc9-4eec-bc92-e3bd46d4e2ee.json
@@ -5,6 +5,12 @@
"owner": "ADMIN",
"join_tables": [],
"all_named_columns": [
+ {
+ "id": 1,
+ "name": "TEST_KYLIN_FACT_CAL_DT",
+ "column": "TEST_KYLIN_FACT_WITH_INT_DATE.CAL_DT",
+ "status": "TOMB"
+ },
{
"id": 2,
"name": "TEST_KYLIN_FACT_ORDER_ID",
diff --git a/src/second-storage/clickhouse/src/main/java/io/kyligence/kap/clickhouse/job/DataLoader.java b/src/second-storage/clickhouse/src/main/java/io/kyligence/kap/clickhouse/job/DataLoader.java
index a9dd394df2..ae3e2b9905 100644
--- a/src/second-storage/clickhouse/src/main/java/io/kyligence/kap/clickhouse/job/DataLoader.java
+++ b/src/second-storage/clickhouse/src/main/java/io/kyligence/kap/clickhouse/job/DataLoader.java
@@ -43,6 +43,7 @@ import io.kyligence.kap.secondstorage.util.SecondStorageDateUtils;
import lombok.Getter;
import lombok.val;
import lombok.extern.slf4j.Slf4j;
+import org.apache.kylin.metadata.model.NDataModel;
@Slf4j
public class DataLoader {
@@ -182,8 +183,9 @@ public class DataLoader {
if (isIncremental) {
String partitionColName = loadInfo.model.getPartitionDesc().getPartitionDateColumn();
val dateCol = loadInfo.model.getAllNamedColumns().stream()
- .filter(column -> column.getAliasDotColumn().equals(partitionColName)).findFirst()
- .orElseThrow(
+ .filter(column -> column.getAliasDotColumn().equals(partitionColName)
+ && NDataModel.ColumnStatus.DIMENSION.equals(column.getStatus()))
+ .findFirst().orElseThrow(
() -> new IllegalStateException("can't find partition column " + partitionColName));
Preconditions.checkState(loadInfo.getLayout().getColumns().stream()
.anyMatch(col -> col.getAliasDotName().equals(dateCol.getAliasDotColumn())));
[kylin] 01/17: [DIRTY] format code
Posted by xx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit fbee49883d02d590eeb388e8979c47eb1e993084
Author: Yinghao Lin <yi...@kyligence.io>
AuthorDate: Tue Dec 20 23:51:12 2022 +0800
[DIRTY] format code
---
.../service/task/QueryHistoryTaskSchedulerRunnerTest.java | 2 +-
.../rest/service/task/QueryHistoryTaskSchedulerTest.java | 6 +++---
.../org/apache/kylin/query/engine/QueryRoutingEngine.java | 7 +++----
.../test/java/org/apache/kylin/query/util/QueryUtilTest.java | 12 ++++--------
src/server/src/main/resources/application.yaml | 2 +-
5 files changed, 12 insertions(+), 17 deletions(-)
diff --git a/src/common-service/src/test/java/org/apache/kylin/rest/service/task/QueryHistoryTaskSchedulerRunnerTest.java b/src/common-service/src/test/java/org/apache/kylin/rest/service/task/QueryHistoryTaskSchedulerRunnerTest.java
index 64a6aa44b9..564901f2e7 100644
--- a/src/common-service/src/test/java/org/apache/kylin/rest/service/task/QueryHistoryTaskSchedulerRunnerTest.java
+++ b/src/common-service/src/test/java/org/apache/kylin/rest/service/task/QueryHistoryTaskSchedulerRunnerTest.java
@@ -118,4 +118,4 @@ public class QueryHistoryTaskSchedulerRunnerTest extends NLocalFileMetadataTestC
}
}
-}
+}
\ No newline at end of file
diff --git a/src/common-service/src/test/java/org/apache/kylin/rest/service/task/QueryHistoryTaskSchedulerTest.java b/src/common-service/src/test/java/org/apache/kylin/rest/service/task/QueryHistoryTaskSchedulerTest.java
index 18e2a86842..270718d3b2 100644
--- a/src/common-service/src/test/java/org/apache/kylin/rest/service/task/QueryHistoryTaskSchedulerTest.java
+++ b/src/common-service/src/test/java/org/apache/kylin/rest/service/task/QueryHistoryTaskSchedulerTest.java
@@ -631,8 +631,8 @@ public class QueryHistoryTaskSchedulerTest extends NLocalFileMetadataTestCase {
queryHistory2.setQueryTime(QUERY_TIME);
queryHistory2.setEngineType("NATIVE");
QueryHistoryInfo queryHistoryInfo2 = new QueryHistoryInfo();
- QueryMetrics.RealizationMetrics realizationMetric = new QueryMetrics.RealizationMetrics(LAYOUT3,
- "Agg Index", "error", Lists.newArrayList());
+ QueryMetrics.RealizationMetrics realizationMetric = new QueryMetrics.RealizationMetrics(LAYOUT3, "Agg Index",
+ "error", Lists.newArrayList());
queryHistoryInfo2.setRealizationMetrics(Lists.newArrayList(realizationMetric));
queryHistory2.setQueryHistoryInfo(queryHistoryInfo2);
queryHistory2.setId(11);
@@ -641,4 +641,4 @@ public class QueryHistoryTaskSchedulerTest extends NLocalFileMetadataTestCase {
int startOffset = 0;
-}
+}
\ No newline at end of file
diff --git a/src/query/src/main/java/org/apache/kylin/query/engine/QueryRoutingEngine.java b/src/query/src/main/java/org/apache/kylin/query/engine/QueryRoutingEngine.java
index a7e85c9056..5937a43650 100644
--- a/src/query/src/main/java/org/apache/kylin/query/engine/QueryRoutingEngine.java
+++ b/src/query/src/main/java/org/apache/kylin/query/engine/QueryRoutingEngine.java
@@ -88,8 +88,7 @@ public class QueryRoutingEngine {
try {
return doTransactionEnabled(() -> {
- if (projectKylinConfig.enableReplaceDynamicParams()
- && queryParams.isPrepareStatementWithParams()) {
+ if (projectKylinConfig.enableReplaceDynamicParams() && queryParams.isPrepareStatementWithParams()) {
queryParams.setSql(queryParams.getPrepareSql());
}
@@ -128,7 +127,7 @@ public class QueryRoutingEngine {
} catch (TransactionException e) {
Throwable cause = e.getCause();
if (cause instanceof SQLException && cause.getCause() instanceof KylinException) {
- throw (SQLException)cause;
+ throw (SQLException) cause;
}
if (shouldPushdown(cause, queryParams)) {
return pushDownQuery((SQLException) cause, queryParams);
@@ -137,7 +136,7 @@ public class QueryRoutingEngine {
}
} catch (SQLException e) {
if (e.getCause() instanceof KylinException) {
- if(checkIfRetryQuery(e.getCause())) {
+ if (checkIfRetryQuery(e.getCause())) {
NProjectLoader.removeCache();
return queryWithSqlMassage(queryParams);
} else {
diff --git a/src/query/src/test/java/org/apache/kylin/query/util/QueryUtilTest.java b/src/query/src/test/java/org/apache/kylin/query/util/QueryUtilTest.java
index 2ca8967ae6..684ed8ecba 100644
--- a/src/query/src/test/java/org/apache/kylin/query/util/QueryUtilTest.java
+++ b/src/query/src/test/java/org/apache/kylin/query/util/QueryUtilTest.java
@@ -240,25 +240,21 @@ public class QueryUtilTest extends NLocalFileMetadataTestCase {
@Test
public void testRemoveCommentInSql() {
//test remove comment when last comment is --
- Assert.assertEquals(
- "select sum(ITEM_COUNT)\nfrom TEST_KYLIN_FACT\ngroup by CAL_DT\n" + "order by SELLER_ID",
+ Assert.assertEquals("select sum(ITEM_COUNT)\nfrom TEST_KYLIN_FACT\ngroup by CAL_DT\n" + "order by SELLER_ID",
QueryUtil.removeCommentInSql(
"select sum(ITEM_COUNT) --1 /* 7 */\nfrom TEST_KYLIN_FACT --2 /* 7 */\ngroup by CAL_DT --3 /* 7 */\n"
+ "order by SELLER_ID; --4 /* 7 */\n--5\n/* 7 */\n--6"));
- Assert.assertEquals(
- "select sum(ITEM_COUNT)\nfrom TEST_KYLIN_FACT\ngroup by CAL_DT\n" + "order by SELLER_ID",
+ Assert.assertEquals("select sum(ITEM_COUNT)\nfrom TEST_KYLIN_FACT\ngroup by CAL_DT\n" + "order by SELLER_ID",
QueryUtil.removeCommentInSql(
"select sum(ITEM_COUNT) --1 /* 7 */\nfrom TEST_KYLIN_FACT --2 /* 7 */\ngroup by CAL_DT --3 /* 7 */\n"
+ "order by SELLER_ID --4 /* 7 */\n--5\n/* 7 */\n--6"));
//test remove comment when last comment is /* */
- Assert.assertEquals(
- "select sum(ITEM_COUNT)\nfrom TEST_KYLIN_FACT\ngroup by CAL_DT\n" + "order by SELLER_ID",
+ Assert.assertEquals("select sum(ITEM_COUNT)\nfrom TEST_KYLIN_FACT\ngroup by CAL_DT\n" + "order by SELLER_ID",
QueryUtil.removeCommentInSql(
"select sum(ITEM_COUNT) --1 /* 7 */\nfrom TEST_KYLIN_FACT --2 /* 7 */\ngroup by CAL_DT --3 /* 7 */\n"
+ "order by SELLER_ID; --4 /* 7 */\n--5\n/* 7 */"));
- Assert.assertEquals(
- "select sum(ITEM_COUNT)\nfrom TEST_KYLIN_FACT\ngroup by CAL_DT\n" + "order by SELLER_ID",
+ Assert.assertEquals("select sum(ITEM_COUNT)\nfrom TEST_KYLIN_FACT\ngroup by CAL_DT\n" + "order by SELLER_ID",
QueryUtil.removeCommentInSql(
"select sum(ITEM_COUNT) --1 /* 7 */\nfrom TEST_KYLIN_FACT --2 /* 7 */\ngroup by CAL_DT --3 /* 7 */\n"
+ "order by SELLER_ID --4 /* 7 */\n--5\n/* 7 */"));
diff --git a/src/server/src/main/resources/application.yaml b/src/server/src/main/resources/application.yaml
index 583a28a682..151df227e8 100644
--- a/src/server/src/main/resources/application.yaml
+++ b/src/server/src/main/resources/application.yaml
@@ -130,4 +130,4 @@ spring:
import: optional:${KYLIN_CONF}/kylin.properties
web:
resources:
- static-locations: "file://${KYLIN_HOME}/kystudio/dist"
+ static-locations: "file://${KYLIN_HOME}/kystudio/dist"
\ No newline at end of file
[kylin] 13/17: [DIRTY] upgrade spark version
Posted by xx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 4099e3b508338dc46427108c7760fef281d26cf8
Author: Zhixiong Chen <ch...@apache.org>
AuthorDate: Wed Nov 9 14:18:59 2022 +0800
[DIRTY] upgrade spark version
Co-authored-by: chenzhx <ch...@apache.io>
---
pom.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pom.xml b/pom.xml
index 80e6a6ed9c..ad78eecd6d 100644
--- a/pom.xml
+++ b/pom.xml
@@ -64,7 +64,7 @@
<!-- Spark versions -->
<delta.version>1.2.1</delta.version>
- <spark.version>3.2.0-kylin-4.6.1.0</spark.version>
+ <spark.version>3.2.0-kylin-4.6.2.0-SNAPSHOT</spark.version>
<roaring.version>0.9.2-kylin-r4</roaring.version>
[kylin] 16/17: KYLIN-5399 refactor core-common
Posted by xx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 1a00e9ac57de03d3436e153dbc535a5ee5809cd2
Author: qianhao.zhou <z....@gmail.com>
AuthorDate: Sat Nov 12 20:41:02 2022 +0800
KYLIN-5399 refactor core-common
* fix async query cannot run
* fix newten-job.jar cannot run
* add missing dependency
Co-authored-by: qhzhou <qi...@kyligence.io>
---
pom.xml | 18 ++
src/assembly/pom.xml | 4 +
src/common-service/pom.xml | 4 +
.../apache/kylin/rest/config/MetricsConfig.java | 4 +-
.../kylin/rest/controller/NBasicController.java | 17 +-
.../interceptor/ReloadAuthoritiesInterceptor.java | 5 +-
.../rest/security/NUnauthorisedEntryPoint.java | 3 +-
.../org/apache/kylin/rest/util/SparkUIUtil.java | 3 +-
src/core-common/pom.xml | 222 ++++-----------------
.../org/apache/kylin/common/KylinConfigBase.java | 4 +-
.../AsyncProfilerConstants.java} | 23 +--
.../common/persistence/metadata/jdbc/JdbcUtil.java | 2 +-
.../common/persistence/transaction/UnitOfWork.java | 22 +-
.../java/org/apache/kylin/common/util/Unsafe.java | 8 +-
src/core-job/pom.xml | 16 ++
.../kylin/common/util/SystemInfoCollector.java | 0
.../apache/kylin/metrics/HdfsCapacityMetrics.java | 0
.../kylin/metrics/HdfsCapacityMetricsTest.java | 0
src/core-metadata/pom.xml | 26 +--
.../kylin/metadata/model/ComputedColumnDesc.java | 9 -
.../apache/kylin/metadata/model/FunctionDesc.java | 5 +-
.../metadata/model/schema/ModelEdgeCollector.java | 12 +-
.../org/apache/kylin/source/jdbc/H2Database.java | 2 +-
src/{core-job => core-metrics}/pom.xml | 128 ++++++------
.../kylin/common/metrics/MetricsCategory.java | 0
.../apache/kylin/common/metrics/MetricsConfig.java | 0
.../kylin/common/metrics/MetricsController.java | 0
.../apache/kylin/common/metrics/MetricsGroup.java | 0
.../common/metrics/MetricsInfluxdbReporter.java | 0
.../apache/kylin/common/metrics/MetricsName.java | 0
.../apache/kylin/common/metrics/MetricsObject.java | 0
.../kylin/common/metrics/MetricsObjectType.java | 0
.../kylin/common/metrics/MetricsReporter.java | 0
.../apache/kylin/common/metrics/MetricsTag.java | 0
.../common/metrics/context/ClusterContext.java | 0
.../kylin/common/metrics/context/ClusterInfo.java | 0
.../common/metrics/gauges/QueryRatioGauge.java | 0
.../metrics/prometheus/PrometheusMetrics.java | 0
.../common/metrics/reporter/InfluxdbReporter.java | 2 +-
.../metrics/reporter/ServerModeMetricFilter.java | 0
.../common/metrics/service/InfluxDBInstance.java | 4 +-
.../metrics/service/JobStatusMonitorMetric.java | 0
.../kylin/common/metrics/service/MonitorDao.java | 0
.../common/metrics/service/MonitorMetric.java | 0
.../metrics/service/MonitorMetricOperation.java | 0
.../common/metrics/service/QueryMonitorMetric.java | 0
.../apache/kylin/common/util/InfluxDBUtils.java | 0
.../kylin/common/metric/InfluxDBInstanceTest.java | 0
.../kylin/common/metric/InfluxMetricsTest.java | 0
.../kylin/common/metric/MetricsControllerTest.java | 16 +-
.../apache/kylin/common/metric/MonitorDaoTest.java | 0
.../kylin/rest/controller/BaseController.java | 19 +-
.../kylin/rest/filter/SegmentsRequestFilter.java | 3 +-
src/datasource-sdk/pom.xml | 27 +--
src/datasource-service/pom.xml | 4 +
src/{core-job => distributed-lock-ext}/pom.xml | 107 +++++-----
.../lock/curator/CuratorDistributedLock.java | 0
.../curator/CuratorDistributedLockFactory.java | 0
.../lock/jdbc/JdbcDistributedLockFactory.java | 0
.../common/lock/jdbc/JdbcDistributedLockUtil.java | 24 ++-
.../java/org/apache/kylin/common/util/ZKUtil.java | 8 +-
.../common/lock/DistributedLockFactoryTest.java | 11 +-
.../curator/CuratorDistributedLockFactoryTest.java | 43 ++--
.../lock/jdbc/JdbcDistributedLockFactoryTest.java | 13 +-
src/kylin-it/pom.xml | 5 +
.../kylin/streaming/StreamingMergeEntryTest.java | 9 +
.../kylin/common/util/AddTableNameSqlVisitor.java | 0
.../common/util/ModifyTableNameSqlVisitor.java | 0
.../kylin/rest/service/ModelSemanticHelper.java | 9 +-
src/query/pom.xml | 4 +
src/server/pom.xml | 24 +++
.../org/apache/kylin/rest/QueryNodeFilter.java | 7 +-
src/spark-project/engine-build-sdk/pom.xml | 1 -
src/spark-project/engine-spark/pom.xml | 4 +
.../kylin/engine/spark/job/JobManagerTest.java | 29 +--
src/spark-project/spark-common/pom.xml | 5 +
.../kylin/common/asyncprofiler/AsyncArchUtil.java | 0
.../kylin/common/asyncprofiler/AsyncProfiler.java | 12 --
.../common/asyncprofiler/AsyncProfilerUtils.java | 0
.../org/apache/kylin/source/dfs/DFSFileTable.java | 0
.../kylin/source/dfs/DFSFileTableReader.java | 4 +-
.../libasyncProfiler-linux-arm64.so | Bin 0 -> 298256 bytes
.../libasyncProfiler-linux-x64.so | Bin 0 -> 303903 bytes
.../async-profiler-lib/libasyncProfiler-mac.so | Bin 0 -> 634272 bytes
.../async-profiler-lib/linux64/libasyncProfiler.so | Bin 0 -> 314098 bytes
.../async-profiler-lib/macOS/libasyncProfiler.so | Bin 0 -> 239460 bytes
.../AsyncProfilerExecutorPlugin.scala | 0
.../common/asyncprofiler/AsyncProfilerTool.scala | 2 +-
.../kylin/common/asyncprofiler/Message.scala | 4 +-
.../common/asyncprofiler/AsyncArchUtilTest.java | 0
.../common/asyncprofiler/AsyncProfilerTest.java | 18 --
.../asyncprofiler/AsyncProfilerToolTest.java | 0
.../asyncprofiler/AsyncProfilerUtilsTest.java | 6 +-
.../kylin/common/asyncprofiler/MessageTest.java | 3 +-
src/systools/pom.xml | 22 +-
95 files changed, 425 insertions(+), 561 deletions(-)
diff --git a/pom.xml b/pom.xml
index ad78eecd6d..40eb6b25f9 100644
--- a/pom.xml
+++ b/pom.xml
@@ -317,6 +317,8 @@
<module>src/query</module>
<module>src/core-common</module>
<module>src/core-metadata</module>
+ <module>src/distributed-lock-ext</module>
+ <module>src/core-metrics</module>
<module>src/source-hive</module>
<module>src/server</module>
<module>src/spark-project/kylin-soft-affinity-cache</module>
@@ -523,6 +525,22 @@
<artifactId>kylin-integration-service</artifactId>
<version>${project.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.kylin</groupId>
+ <artifactId>distributed-lock-ext</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.kylin</groupId>
+ <artifactId>kylin-core-metrics</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.kylin</groupId>
+ <artifactId>kylin-streaming</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ </dependency>
<!-- arthas -->
<dependency>
diff --git a/src/assembly/pom.xml b/src/assembly/pom.xml
index 2a56110bfc..5436fda9bd 100644
--- a/src/assembly/pom.xml
+++ b/src/assembly/pom.xml
@@ -88,6 +88,10 @@
<groupId>org.apache.kylin</groupId>
<artifactId>kylin-query</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.postgresql</groupId>
+ <artifactId>postgresql</artifactId>
+ </dependency>
<!-- Env -->
<dependency>
<groupId>org.apache.hadoop</groupId>
diff --git a/src/common-service/pom.xml b/src/common-service/pom.xml
index 6226d6ea78..d6fcd55cc1 100644
--- a/src/common-service/pom.xml
+++ b/src/common-service/pom.xml
@@ -59,6 +59,10 @@
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
</dependency>
+ <dependency>
+ <groupId>commons-fileupload</groupId>
+ <artifactId>commons-fileupload</artifactId>
+ </dependency>
<dependency>
<groupId>net.sf.ehcache</groupId>
diff --git a/src/common-service/src/main/java/org/apache/kylin/rest/config/MetricsConfig.java b/src/common-service/src/main/java/org/apache/kylin/rest/config/MetricsConfig.java
index a0126eee3e..0289709b62 100644
--- a/src/common-service/src/main/java/org/apache/kylin/rest/config/MetricsConfig.java
+++ b/src/common-service/src/main/java/org/apache/kylin/rest/config/MetricsConfig.java
@@ -28,11 +28,11 @@ import java.util.stream.Collectors;
import org.apache.kylin.common.KapConfig;
import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.NamedThreadFactory;
-import org.apache.kylin.metadata.project.ProjectInstance;
import org.apache.kylin.common.metrics.MetricsController;
import org.apache.kylin.common.metrics.MetricsGroup;
+import org.apache.kylin.common.util.NamedThreadFactory;
import org.apache.kylin.metadata.project.NProjectManager;
+import org.apache.kylin.metadata.project.ProjectInstance;
import org.apache.kylin.rest.cluster.ClusterManager;
import org.apache.kylin.rest.config.initialize.MetricsRegistry;
import org.springframework.beans.factory.annotation.Autowired;
diff --git a/src/common-service/src/main/java/org/apache/kylin/rest/controller/NBasicController.java b/src/common-service/src/main/java/org/apache/kylin/rest/controller/NBasicController.java
index 4ffe05c013..884a807128 100644
--- a/src/common-service/src/main/java/org/apache/kylin/rest/controller/NBasicController.java
+++ b/src/common-service/src/main/java/org/apache/kylin/rest/controller/NBasicController.java
@@ -83,7 +83,6 @@ import org.apache.kylin.common.msg.MsgPicker;
import org.apache.kylin.common.persistence.transaction.TransactionException;
import org.apache.kylin.common.util.DateFormat;
import org.apache.kylin.common.util.JsonUtil;
-import org.apache.kylin.common.util.Unsafe;
import org.apache.kylin.job.constant.JobStatusEnum;
import org.apache.kylin.job.dao.ExecutablePO;
import org.apache.kylin.job.execution.JobTypeEnum;
@@ -188,7 +187,7 @@ public class NBasicController {
if (kylinException != null) {
cause = kylinException;
}
- return new ErrorResponse(Unsafe.getUrlFromHttpServletRequest(req), cause);
+ return new ErrorResponse(req.getRequestURL().toString(), cause);
}
@ResponseStatus(HttpStatus.FORBIDDEN)
@@ -196,7 +195,7 @@ public class NBasicController {
@ResponseBody
ErrorResponse handleForbidden(HttpServletRequest req, Exception ex) {
getLogger().error("", ex);
- return new ErrorResponse(Unsafe.getUrlFromHttpServletRequest(req), ex);
+ return new ErrorResponse(req.getRequestURL().toString(), ex);
}
@ResponseStatus(HttpStatus.NOT_FOUND)
@@ -204,7 +203,7 @@ public class NBasicController {
@ResponseBody
ErrorResponse handleNotFound(HttpServletRequest req, Exception ex) {
getLogger().error("", ex);
- return new ErrorResponse(Unsafe.getUrlFromHttpServletRequest(req), ex);
+ return new ErrorResponse(req.getRequestURL().toString(), ex);
}
@ResponseStatus(HttpStatus.INTERNAL_SERVER_ERROR)
@@ -228,7 +227,7 @@ public class NBasicController {
ErrorResponse handleAccessDenied(HttpServletRequest req, Throwable ex) {
getLogger().error("", ex);
KylinException e = new KylinException(ACCESS_DENIED, MsgPicker.getMsg().getAccessDeny());
- return new ErrorResponse(Unsafe.getUrlFromHttpServletRequest(req), e);
+ return new ErrorResponse(req.getRequestURL().toString(), e);
}
@ResponseStatus(HttpStatus.BAD_REQUEST)
@@ -238,7 +237,7 @@ public class NBasicController {
ErrorResponse handleInvalidRequestParam(HttpServletRequest req, Throwable ex) {
KylinException e = new KylinException(INVALID_PARAMETER, ex);
getLogger().error("", e);
- return new ErrorResponse(Unsafe.getUrlFromHttpServletRequest(req), e);
+ return new ErrorResponse(req.getRequestURL().toString(), e);
}
@ResponseStatus(HttpStatus.INTERNAL_SERVER_ERROR)
@@ -247,14 +246,14 @@ public class NBasicController {
ErrorResponse handleErrorCode(HttpServletRequest req, Throwable ex) {
getLogger().error("", ex);
KylinException cause = (KylinException) ex;
- return new ErrorResponse(Unsafe.getUrlFromHttpServletRequest(req), cause);
+ return new ErrorResponse(req.getRequestURL().toString(), cause);
}
@ResponseStatus(HttpStatus.BAD_REQUEST)
@ExceptionHandler(MethodArgumentNotValidException.class)
@ResponseBody
ErrorResponse handleInvalidArgument(HttpServletRequest request, MethodArgumentNotValidException ex) {
- val response = new ErrorResponse(Unsafe.getUrlFromHttpServletRequest(request), ex);
+ val response = new ErrorResponse(request.getRequestURL().toString(), ex);
val target = ex.getBindingResult().getTarget();
if (target instanceof Validation) {
response.setMsg(((Validation) target).getErrorMessage(ex.getBindingResult().getFieldErrors()));
@@ -272,7 +271,7 @@ public class NBasicController {
ErrorResponse handleUnauthorized(HttpServletRequest req, Throwable ex) {
KylinException e = new KylinException(USER_UNAUTHORIZED, ex);
getLogger().error("", e);
- return new ErrorResponse(Unsafe.getUrlFromHttpServletRequest(req), ex);
+ return new ErrorResponse(req.getRequestURL().toString(), ex);
}
protected void checkRequiredArg(String fieldName, Object fieldValue) {
diff --git a/src/common-service/src/main/java/org/apache/kylin/rest/interceptor/ReloadAuthoritiesInterceptor.java b/src/common-service/src/main/java/org/apache/kylin/rest/interceptor/ReloadAuthoritiesInterceptor.java
index 0d8714647a..1d48d51d66 100644
--- a/src/common-service/src/main/java/org/apache/kylin/rest/interceptor/ReloadAuthoritiesInterceptor.java
+++ b/src/common-service/src/main/java/org/apache/kylin/rest/interceptor/ReloadAuthoritiesInterceptor.java
@@ -31,10 +31,9 @@ import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.exception.KylinException;
import org.apache.kylin.common.msg.MsgPicker;
import org.apache.kylin.common.util.JsonUtil;
+import org.apache.kylin.metadata.user.ManagedUser;
import org.apache.kylin.rest.response.ErrorResponse;
import org.apache.kylin.rest.service.UserService;
-import org.apache.kylin.common.util.Unsafe;
-import org.apache.kylin.metadata.user.ManagedUser;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.core.annotation.Order;
@@ -84,7 +83,7 @@ public class ReloadAuthoritiesInterceptor extends HandlerInterceptorAdapter {
if (present) {
response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
response.setContentType(MediaType.APPLICATION_JSON_VALUE);
- ErrorResponse errorResponse = new ErrorResponse(Unsafe.getUrlFromHttpServletRequest(request),
+ ErrorResponse errorResponse = new ErrorResponse(request.getRequestURL().toString(),
new KylinException(USER_DATA_SOURCE_CONNECTION_FAILED,
MsgPicker.getMsg().getlDapUserDataSourceConnectionFailed()));
response.setCharacterEncoding("UTF-8");
diff --git a/src/common-service/src/main/java/org/apache/kylin/rest/security/NUnauthorisedEntryPoint.java b/src/common-service/src/main/java/org/apache/kylin/rest/security/NUnauthorisedEntryPoint.java
index 6c376b81f7..3aceb2ad5c 100644
--- a/src/common-service/src/main/java/org/apache/kylin/rest/security/NUnauthorisedEntryPoint.java
+++ b/src/common-service/src/main/java/org/apache/kylin/rest/security/NUnauthorisedEntryPoint.java
@@ -37,7 +37,6 @@ import org.apache.kylin.common.exception.KylinException;
import org.apache.kylin.common.msg.MsgPicker;
import org.apache.kylin.common.util.JsonUtil;
import org.apache.kylin.rest.response.ErrorResponse;
-import org.apache.kylin.common.util.Unsafe;
import org.springframework.http.MediaType;
import org.springframework.ldap.CommunicationException;
import org.springframework.security.authentication.DisabledException;
@@ -94,7 +93,7 @@ public class NUnauthorisedEntryPoint implements AuthenticationEntryPoint {
throws IOException {
response.setStatus(statusCode);
response.setContentType(MediaType.APPLICATION_JSON_VALUE);
- ErrorResponse errorResponse = new ErrorResponse(Unsafe.getUrlFromHttpServletRequest(request), ex);
+ ErrorResponse errorResponse = new ErrorResponse(request.getRequestURL().toString(), ex);
String errorStr = JsonUtil.writeValueAsIndentString(errorResponse);
response.setCharacterEncoding("UTF-8");
PrintWriter writer = response.getWriter();
diff --git a/src/common-service/src/main/java/org/apache/kylin/rest/util/SparkUIUtil.java b/src/common-service/src/main/java/org/apache/kylin/rest/util/SparkUIUtil.java
index a814b725d0..51a01d6c1e 100644
--- a/src/common-service/src/main/java/org/apache/kylin/rest/util/SparkUIUtil.java
+++ b/src/common-service/src/main/java/org/apache/kylin/rest/util/SparkUIUtil.java
@@ -28,7 +28,6 @@ import javax.servlet.http.HttpServletResponse;
import org.apache.commons.io.IOUtils;
import org.apache.http.impl.client.HttpClientBuilder;
-import org.apache.kylin.common.util.Unsafe;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.http.HttpHeaders;
@@ -59,7 +58,7 @@ public class SparkUIUtil {
final HttpMethod method = HttpMethod.resolve(servletRequest.getMethod());
try (ClientHttpResponse response = execute(target, method, proxyLocationBase)) {
- rewrite(response, servletResponse, method, Unsafe.getUrlFromHttpServletRequest(servletRequest),
+ rewrite(response, servletResponse, method, servletRequest.getRequestURL().toString(),
REDIRECT_THRESHOLD, proxyLocationBase);
}
}
diff --git a/src/core-common/pom.xml b/src/core-common/pom.xml
index 6c1211efe8..47bd12746c 100644
--- a/src/core-common/pom.xml
+++ b/src/core-common/pom.xml
@@ -37,9 +37,8 @@
<dependencies>
<dependency>
<groupId>io.kyligence.ke</groupId>
- <artifactId>kap-external-curator</artifactId>
+ <artifactId>kap-external-guava20</artifactId>
</dependency>
-
<dependency>
<groupId>io.kyligence</groupId>
<artifactId>kyligence-config-external</artifactId>
@@ -50,67 +49,25 @@
</exclusion>
</exclusions>
</dependency>
-
- <!-- provided -->
- <dependency>
- <groupId>io.netty</groupId>
- <artifactId>netty</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-hdfs</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.tomcat.embed</groupId>
- <artifactId>tomcat-embed-core</artifactId>
- </dependency>
-
- <dependency>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-lang3</artifactId>
- </dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-email</artifactId>
</dependency>
- <dependency>
- <groupId>commons-fileupload</groupId>
- <artifactId>commons-fileupload</artifactId>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-databind</artifactId>
- </dependency>
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
</dependency>
- <dependency>
- <groupId>com.lmax</groupId>
- <artifactId>disruptor</artifactId>
- </dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-api</artifactId>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-core</artifactId>
+ <artifactId>log4j-web</artifactId>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-web</artifactId>
+ <artifactId>log4j-core</artifactId>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
@@ -121,69 +78,42 @@
<artifactId>log4j-slf4j-impl</artifactId>
</dependency>
<dependency>
- <groupId>org.mybatis</groupId>
- <artifactId>mybatis</artifactId>
+ <groupId>javax.validation</groupId>
+ <artifactId>validation-api</artifactId>
</dependency>
<dependency>
- <groupId>org.mybatis.dynamic-sql</groupId>
- <artifactId>mybatis-dynamic-sql</artifactId>
- </dependency>
-
- <!-- dropwizard -->
- <dependency>
- <groupId>io.dropwizard.metrics</groupId>
- <artifactId>metrics-core</artifactId>
- </dependency>
- <dependency>
- <groupId>io.dropwizard.metrics</groupId>
- <artifactId>metrics-jvm</artifactId>
- </dependency>
- <dependency>
- <groupId>io.dropwizard.metrics</groupId>
- <artifactId>metrics-json</artifactId>
- </dependency>
- <dependency>
- <groupId>io.dropwizard.metrics</groupId>
- <artifactId>metrics-jmx</artifactId>
- </dependency>
-
- <!-- micrometer -->
- <dependency>
- <groupId>io.micrometer</groupId>
- <artifactId>micrometer-registry-prometheus</artifactId>
+ <groupId>com.alibaba</groupId>
+ <artifactId>transmittable-thread-local</artifactId>
</dependency>
-
<dependency>
- <groupId>org.latencyutils</groupId>
- <artifactId>LatencyUtils</artifactId>
+ <groupId>org.springframework</groupId>
+ <artifactId>spring-jdbc</artifactId>
+ <exclusions>
+ <exclusion>
+ <groupId>org.springframework</groupId>
+ <artifactId>spring-jcl</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
-
+ <!-- provided -->
<dependency>
- <groupId>io.kyligence.ke</groupId>
- <artifactId>kap-external-guava20</artifactId>
+ <groupId>org.springframework.security</groupId>
+ <artifactId>spring-security-core</artifactId>
+ <scope>provided</scope>
</dependency>
<dependency>
- <groupId>io.kyligence.ke</groupId>
- <artifactId>kap-external-influxdb</artifactId>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-dbcp2</artifactId>
+ <scope>provided</scope>
</dependency>
+ <!-- test -->
<dependency>
<groupId>joda-time</groupId>
<artifactId>joda-time</artifactId>
<scope>test</scope>
<version>2.10.2</version>
</dependency>
-
- <dependency>
- <groupId>com.github.oshi</groupId>
- <artifactId>oshi-core</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.javassist</groupId>
- <artifactId>javassist</artifactId>
- </dependency>
- <!--Env & Test-->
<dependency>
<groupId>com.github.sakserv</groupId>
<artifactId>hadoop-mini-clusters-hdfs</artifactId>
@@ -191,25 +121,8 @@
<version>0.1.16</version>
</dependency>
<dependency>
- <groupId>mysql</groupId>
- <artifactId>mysql-connector-java</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.postgresql</groupId>
- <artifactId>postgresql</artifactId>
- </dependency>
-
- <!-- postgresql 42.2.25 not dependency jcl-over-slf4j, ut testFindContainingJar will fail -->
- <dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>jcl-over-slf4j</artifactId>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.curator</groupId>
- <artifactId>curator-test</artifactId>
+ <groupId>org.assertj</groupId>
+ <artifactId>assertj-core</artifactId>
<scope>test</scope>
</dependency>
<dependency>
@@ -217,43 +130,6 @@
<artifactId>spring-test</artifactId>
<scope>test</scope>
</dependency>
- <dependency>
- <groupId>commons-io</groupId>
- <artifactId>commons-io</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.kafka</groupId>
- <artifactId>kafka_2.12</artifactId>
- <scope>provided</scope>
- </dependency>
-
- <!-- jdbc persistence -->
- <dependency>
- <groupId>org.springframework</groupId>
- <artifactId>spring-jdbc</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.springframework</groupId>
- <artifactId>spring-context-support</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.springframework.security</groupId>
- <artifactId>spring-security-core</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-dbcp2</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.assertj</groupId>
- <artifactId>assertj-core</artifactId>
- </dependency>
-
<dependency>
<groupId>org.awaitility</groupId>
<artifactId>awaitility</artifactId>
@@ -274,11 +150,6 @@
<artifactId>junit-pioneer</artifactId>
<scope>test</scope>
</dependency>
- <dependency>
- <groupId>com.h2database</groupId>
- <artifactId>h2</artifactId>
- <scope>test</scope>
- </dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-core</artifactId>
@@ -289,7 +160,6 @@
<artifactId>log4j-core</artifactId>
</exclusion>
</exclusions>
- <version>${mockito.version}</version>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
@@ -301,7 +171,6 @@
<artifactId>log4j-core</artifactId>
</exclusion>
</exclusions>
- <version>${mockito.version}</version>
</dependency>
<dependency>
<groupId>org.powermock</groupId>
@@ -314,32 +183,15 @@
<scope>test</scope>
</dependency>
<dependency>
- <groupId>commons-codec</groupId>
- <artifactId>commons-codec</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.calcite</groupId>
- <artifactId>calcite-core</artifactId>
- </dependency>
- <dependency>
- <groupId>com.alibaba</groupId>
- <artifactId>transmittable-thread-local</artifactId>
- </dependency>
- <dependency>
- <groupId>org.springframework.cloud</groupId>
- <artifactId>spring-cloud-commons</artifactId>
- </dependency>
- <dependency>
- <groupId>org.springframework.integration</groupId>
- <artifactId>spring-integration-jdbc</artifactId>
+ <groupId>com.h2database</groupId>
+ <artifactId>h2</artifactId>
+ <scope>test</scope>
</dependency>
-
- <!-- Spark -->
<dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-core_2.12</artifactId>
+ <groupId>org.slf4j</groupId>
+ <artifactId>jcl-over-slf4j</artifactId>
+ <scope>test</scope>
</dependency>
-
</dependencies>
<build>
<plugins>
@@ -374,6 +226,18 @@
<groupId>net.alchim31.maven</groupId>
<artifactId>scala-maven-plugin</artifactId>
</plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-jar-plugin</artifactId>
+ <version>3.0.2</version>
+ <executions>
+ <execution>
+ <goals>
+ <goal>test-jar</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
</plugins>
</build>
</project>
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 557901c837..7bd4da912a 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -19,8 +19,8 @@
package org.apache.kylin.common;
import static java.lang.Math.toIntExact;
-import static org.apache.kylin.common.asyncprofiler.AsyncProfiler.ASYNC_PROFILER_LIB_LINUX_ARM64;
-import static org.apache.kylin.common.asyncprofiler.AsyncProfiler.ASYNC_PROFILER_LIB_LINUX_X64;
+import static org.apache.kylin.common.constant.AsyncProfilerConstants.ASYNC_PROFILER_LIB_LINUX_ARM64;
+import static org.apache.kylin.common.constant.AsyncProfilerConstants.ASYNC_PROFILER_LIB_LINUX_X64;
import static org.apache.kylin.common.constant.Constants.KYLIN_SOURCE_JDBC_CONNECTION_URL_KEY;
import static org.apache.kylin.common.constant.Constants.KYLIN_SOURCE_JDBC_DRIVER_KEY;
import static org.apache.kylin.common.constant.Constants.KYLIN_SOURCE_JDBC_PASS_KEY;
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/metrics/MetricsObjectType.java b/src/core-common/src/main/java/org/apache/kylin/common/constant/AsyncProfilerConstants.java
similarity index 66%
copy from src/core-common/src/main/java/org/apache/kylin/common/metrics/MetricsObjectType.java
copy to src/core-common/src/main/java/org/apache/kylin/common/constant/AsyncProfilerConstants.java
index 1502b72de5..1949f0db98 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/metrics/MetricsObjectType.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/constant/AsyncProfilerConstants.java
@@ -15,25 +15,14 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+package org.apache.kylin.common.constant;
-package org.apache.kylin.common.metrics;
+public final class AsyncProfilerConstants {
-public enum MetricsObjectType {
+ private AsyncProfilerConstants() {}
- MAX("max"),
- MIN("min"),
- COUNT("count"),
- FIRST("first"),
- LAST("last");
-
- private String type;
-
- MetricsObjectType(String type) {
- this.type = type;
- }
-
- public String getVal() {
- return this.type;
- }
+ public static final String ASYNC_PROFILER_LIB_MAC = "libasyncProfiler-mac.so";
+ public static final String ASYNC_PROFILER_LIB_LINUX_X64 = "libasyncProfiler-linux-x64.so";
+ public static final String ASYNC_PROFILER_LIB_LINUX_ARM64 = "libasyncProfiler-linux-arm64.so";
}
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/persistence/metadata/jdbc/JdbcUtil.java b/src/core-common/src/main/java/org/apache/kylin/common/persistence/metadata/jdbc/JdbcUtil.java
index 94734b415e..caf500bc14 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/persistence/metadata/jdbc/JdbcUtil.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/persistence/metadata/jdbc/JdbcUtil.java
@@ -36,7 +36,6 @@ import org.apache.kylin.common.exception.KylinException;
import org.apache.kylin.common.msg.MsgPicker;
import org.apache.kylin.common.persistence.metadata.PersistException;
import org.apache.kylin.common.util.EncryptUtil;
-import org.msgpack.core.annotations.VisibleForTesting;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.dao.DataIntegrityViolationException;
@@ -45,6 +44,7 @@ import org.springframework.jdbc.datasource.DataSourceTransactionManager;
import org.springframework.transaction.TransactionDefinition;
import org.springframework.transaction.support.DefaultTransactionDefinition;
+import io.kyligence.kap.guava20.shaded.common.annotations.VisibleForTesting;
import lombok.val;
import lombok.extern.slf4j.Slf4j;
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/persistence/transaction/UnitOfWork.java b/src/core-common/src/main/java/org/apache/kylin/common/persistence/transaction/UnitOfWork.java
index 280e1987d7..4e576d4e4f 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/persistence/transaction/UnitOfWork.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/persistence/transaction/UnitOfWork.java
@@ -18,7 +18,6 @@
package org.apache.kylin.common.persistence.transaction;
import java.util.List;
-import java.util.Map;
import java.util.Objects;
import java.util.function.Consumer;
import java.util.stream.Collectors;
@@ -31,11 +30,6 @@ import org.apache.kylin.common.persistence.RawResource;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.persistence.ThreadViewResourceStore;
import org.apache.kylin.common.persistence.TombRawResource;
-import org.apache.kylin.common.util.Pair;
-import org.apache.kylin.common.util.RandomUtil;
-import org.apache.kylin.common.metrics.MetricsCategory;
-import org.apache.kylin.common.metrics.MetricsGroup;
-import org.apache.kylin.common.metrics.MetricsName;
import org.apache.kylin.common.persistence.UnitMessages;
import org.apache.kylin.common.persistence.event.EndUnit;
import org.apache.kylin.common.persistence.event.Event;
@@ -44,6 +38,8 @@ import org.apache.kylin.common.persistence.event.ResourceDeleteEvent;
import org.apache.kylin.common.persistence.event.ResourceRelatedEvent;
import org.apache.kylin.common.persistence.event.StartUnit;
import org.apache.kylin.common.scheduler.EventBusFactory;
+import org.apache.kylin.common.util.Pair;
+import org.apache.kylin.common.util.RandomUtil;
import org.apache.kylin.common.util.Unsafe;
import com.google.common.base.Preconditions;
@@ -100,17 +96,6 @@ public class UnitOfWork {
int retry = 0;
val traceId = RandomUtil.randomUUIDStr();
while (retry++ < maxRetry) {
- if (retry > 1) {
- Map<String, String> tags = MetricsGroup.getHostTagMap(params.getUnitName());
-
- if (!GLOBAL_UNIT.equals(params.getUnitName())) {
- MetricsGroup.counterInc(MetricsName.TRANSACTION_RETRY_COUNTER, MetricsCategory.PROJECT,
- params.getUnitName(), tags);
- } else {
- MetricsGroup.counterInc(MetricsName.TRANSACTION_RETRY_COUNTER, MetricsCategory.GLOBAL, "global",
- tags);
- }
- }
val ret = doTransaction(params, retry, traceId);
if (ret.getSecond()) {
@@ -147,9 +132,6 @@ public class UnitOfWork {
long duration = System.currentTimeMillis() - startTransactionTime;
logIfLongTransaction(duration, traceId);
- MetricsGroup.hostTagHistogramUpdate(MetricsName.TRANSACTION_LATENCY, MetricsCategory.PROJECT,
- !GLOBAL_UNIT.equals(params.getUnitName()) ? params.getUnitName() : "global", duration);
-
result = Pair.newPair(ret, true);
} catch (Throwable throwable) {
handleError(throwable, params, retry, traceId);
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/util/Unsafe.java b/src/core-common/src/main/java/org/apache/kylin/common/util/Unsafe.java
index 552b9edc44..cfd276f15d 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/util/Unsafe.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/util/Unsafe.java
@@ -24,12 +24,10 @@ import java.text.MessageFormat;
import java.util.Locale;
import java.util.Map;
-import javax.servlet.http.HttpServletRequest;
-
import org.apache.commons.lang.StringUtils;
+import org.apache.kylin.common.SystemPropertiesCache;
import lombok.extern.slf4j.Slf4j;
-import org.apache.kylin.common.SystemPropertiesCache;
/**
* Contains methods that call JDK methods that the
@@ -81,10 +79,6 @@ public class Unsafe {
return temp.format(arguments);
}
- public static String getUrlFromHttpServletRequest(HttpServletRequest request) {
- return request.getRequestURL().toString();
- }
-
/** Reflection usage to work around access flags fails with SecurityManagers
* and likely will not work anymore on runtime classes in Java 9 */
public static void changeAccessibleObject(AccessibleObject accessibleObject, boolean value) {
diff --git a/src/core-job/pom.xml b/src/core-job/pom.xml
index bbdbb0cf2f..a5ac6dff24 100644
--- a/src/core-job/pom.xml
+++ b/src/core-job/pom.xml
@@ -45,6 +45,18 @@
<groupId>io.kyligence.ke</groupId>
<artifactId>kap-external-curator-test</artifactId>
</dependency>
+ <dependency>
+ <groupId>io.kyligence.ke</groupId>
+ <artifactId>kap-external-curator</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.kylin</groupId>
+ <artifactId>kylin-core-metrics</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.kylin</groupId>
+ <artifactId>kylin-core-metrics</artifactId>
+ </dependency>
<dependency>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin-core-common</artifactId>
@@ -60,6 +72,10 @@
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
</dependency>
+ <dependency>
+ <groupId>com.github.oshi</groupId>
+ <artifactId>oshi-core</artifactId>
+ </dependency>
<dependency>
<groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/util/SystemInfoCollector.java b/src/core-job/src/main/java/org/apache/kylin/common/util/SystemInfoCollector.java
similarity index 100%
rename from src/core-common/src/main/java/org/apache/kylin/common/util/SystemInfoCollector.java
rename to src/core-job/src/main/java/org/apache/kylin/common/util/SystemInfoCollector.java
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metrics/HdfsCapacityMetrics.java b/src/core-job/src/main/java/org/apache/kylin/metrics/HdfsCapacityMetrics.java
similarity index 100%
rename from src/core-metadata/src/main/java/org/apache/kylin/metrics/HdfsCapacityMetrics.java
rename to src/core-job/src/main/java/org/apache/kylin/metrics/HdfsCapacityMetrics.java
diff --git a/src/core-metadata/src/test/java/org/apache/kylin/metrics/HdfsCapacityMetricsTest.java b/src/core-job/src/test/java/org/apache/kylin/metrics/HdfsCapacityMetricsTest.java
similarity index 100%
rename from src/core-metadata/src/test/java/org/apache/kylin/metrics/HdfsCapacityMetricsTest.java
rename to src/core-job/src/test/java/org/apache/kylin/metrics/HdfsCapacityMetricsTest.java
diff --git a/src/core-metadata/pom.xml b/src/core-metadata/pom.xml
index 96760a701c..d61c5ab80e 100644
--- a/src/core-metadata/pom.xml
+++ b/src/core-metadata/pom.xml
@@ -36,22 +36,17 @@
</properties>
<dependencies>
-
- <!-- provided -->
<dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- <scope>provided</scope>
+ <groupId>commons-collections</groupId>
+ <artifactId>commons-collections</artifactId>
</dependency>
-
<dependency>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin-core-common</artifactId>
</dependency>
-
<dependency>
- <groupId>org.springframework</groupId>
- <artifactId>spring-jdbc</artifactId>
+ <groupId>io.kyligence.ke</groupId>
+ <artifactId>kap-external-influxdb</artifactId>
</dependency>
<dependency>
<groupId>com.tdunning</groupId>
@@ -65,17 +60,13 @@
<groupId>org.apache.commons</groupId>
<artifactId>commons-dbcp2</artifactId>
</dependency>
- <dependency>
- <groupId>commons-lang</groupId>
- <artifactId>commons-lang</artifactId>
- </dependency>
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
</dependency>
<dependency>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-lang3</artifactId>
</dependency>
<dependency>
<groupId>com.esotericsoftware</groupId>
@@ -109,6 +100,11 @@
<type>test-jar</type>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <scope>test</scope>
+ </dependency>
<dependency>
<groupId>org.junit.vintage</groupId>
<artifactId>junit-vintage-engine</artifactId>
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/ComputedColumnDesc.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/ComputedColumnDesc.java
index 5cceedd5df..f807dea94d 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/ComputedColumnDesc.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/ComputedColumnDesc.java
@@ -32,7 +32,6 @@ import org.apache.calcite.sql.SqlBasicCall;
import org.apache.calcite.sql.SqlCall;
import org.apache.calcite.sql.SqlIdentifier;
import org.apache.calcite.sql.SqlNode;
-import org.apache.calcite.sql.dialect.HiveSqlDialect;
import org.apache.calcite.sql.util.SqlBasicVisitor;
import org.apache.calcite.sql.util.SqlVisitor;
import org.apache.commons.lang.StringUtils;
@@ -40,7 +39,6 @@ import org.apache.kylin.common.exception.KylinException;
import org.apache.kylin.common.msg.MsgPicker;
import org.apache.kylin.measure.MeasureTypeFactory;
import org.apache.kylin.metadata.model.tool.CalciteParser;
-import org.apache.kylin.common.util.ModifyTableNameSqlVisitor;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -211,11 +209,4 @@ public class ComputedColumnDesc implements Serializable {
return innerExpression;
}
- public void changeTableAlias(String oldAlias, String newAlias) {
- SqlVisitor<Object> modifyAlias = new ModifyTableNameSqlVisitor(oldAlias, newAlias);
- SqlNode sqlNode = CalciteParser.getExpNode(getExpression());
- sqlNode.accept(modifyAlias);
- setExpression(sqlNode.toSqlString(HiveSqlDialect.DEFAULT).toString());
- }
-
}
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
index b2a212f84f..28e3603d28 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
@@ -42,8 +42,7 @@ import java.util.Set;
import java.util.stream.Collectors;
import org.apache.commons.collections.CollectionUtils;
-import org.apache.commons.lang.StringUtils;
-import org.apache.directory.api.util.Strings;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.exception.KylinException;
import org.apache.kylin.measure.MeasureType;
import org.apache.kylin.measure.MeasureTypeFactory;
@@ -104,7 +103,7 @@ public class FunctionDesc implements Serializable {
break;
}
case FunctionDesc.FUNC_SUM_LC: {
- Preconditions.checkArgument(Strings.isNotEmpty(colDataType),
+ Preconditions.checkArgument(StringUtils.isNotEmpty(colDataType),
"SUM_LC Measure's input type shouldn't be null or empty");
checkSumLCDataType(colDataType);
break;
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/schema/ModelEdgeCollector.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/schema/ModelEdgeCollector.java
index 8b94d8c7a9..ccda8a9527 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/schema/ModelEdgeCollector.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/schema/ModelEdgeCollector.java
@@ -17,6 +17,7 @@
*/
package org.apache.kylin.metadata.model.schema;
+import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -26,14 +27,14 @@ import java.util.stream.Collectors;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.kylin.common.util.Pair;
-import org.apache.kylin.metadata.model.JoinTableDesc;
-import org.apache.kylin.metadata.model.ParameterDesc;
-import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.cube.cuboid.NAggregationGroup;
import org.apache.kylin.metadata.cube.model.IndexPlan;
import org.apache.kylin.metadata.cube.model.LayoutEntity;
import org.apache.kylin.metadata.model.ComputedColumnDesc;
+import org.apache.kylin.metadata.model.JoinTableDesc;
import org.apache.kylin.metadata.model.NDataModel;
+import org.apache.kylin.metadata.model.ParameterDesc;
+import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.model.util.ComputedColumnUtil;
import com.google.common.collect.ImmutableBiMap;
@@ -41,7 +42,6 @@ import com.google.common.collect.Maps;
import io.kyligence.kap.guava20.shaded.common.graph.Graph;
import io.kyligence.kap.guava20.shaded.common.graph.MutableGraph;
-import io.kyligence.kap.shaded.curator.org.apache.curator.shaded.com.google.common.collect.Lists;
import lombok.NonNull;
import lombok.RequiredArgsConstructor;
import lombok.val;
@@ -79,9 +79,9 @@ class ModelEdgeCollector {
collectModelSignificant();
collectDimensionAndMeasure();
- collectIndex(indexPlan.getWhitelistLayouts(), SchemaNodeType.WHITE_LIST_INDEX, Lists.newArrayList());
+ collectIndex(indexPlan.getWhitelistLayouts(), SchemaNodeType.WHITE_LIST_INDEX, new ArrayList<>());
collectIndex(indexPlan.getToBeDeletedIndexes().stream().flatMap(index -> index.getLayouts().stream())
- .collect(Collectors.toList()), SchemaNodeType.TO_BE_DELETED_INDEX, Lists.newArrayList());
+ .collect(Collectors.toList()), SchemaNodeType.TO_BE_DELETED_INDEX, new ArrayList<>());
collectIndex(indexPlan.getRuleBaseLayouts(), SchemaNodeType.RULE_BASED_INDEX, indexPlan.getAggShardByColumns());
collectAggGroup();
diff --git a/src/source-hive/src/main/java/org/apache/kylin/source/jdbc/H2Database.java b/src/core-metadata/src/main/java/org/apache/kylin/source/jdbc/H2Database.java
similarity index 100%
rename from src/source-hive/src/main/java/org/apache/kylin/source/jdbc/H2Database.java
rename to src/core-metadata/src/main/java/org/apache/kylin/source/jdbc/H2Database.java
index 2df4a82cda..15e933ee13 100644
--- a/src/source-hive/src/main/java/org/apache/kylin/source/jdbc/H2Database.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/source/jdbc/H2Database.java
@@ -36,8 +36,8 @@ import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.metadata.model.ColumnDesc;
-import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.metadata.model.NTableMetadataManager;
+import org.apache.kylin.metadata.model.TableDesc;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/src/core-job/pom.xml b/src/core-metrics/pom.xml
similarity index 51%
copy from src/core-job/pom.xml
copy to src/core-metrics/pom.xml
index bbdbb0cf2f..a7e8742a4e 100644
--- a/src/core-job/pom.xml
+++ b/src/core-metrics/pom.xml
@@ -1,78 +1,83 @@
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
+<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
- <artifactId>kylin-core-job</artifactId>
- <packaging>jar</packaging>
- <name>Kylin - Core Job</name>
- <url>http://kylin.apache.org</url>
- <description>Kylin Core Job</description>
-
<parent>
- <groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
+ <groupId>org.apache.kylin</groupId>
<version>5.0.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
+ <name>Kylin - Core Metrics</name>
+ <url>http://kylin.apache.org</url>
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>kylin-core-metrics</artifactId>
<properties>
+ <maven.compiler.source>8</maven.compiler.source>
+ <maven.compiler.target>8</maven.compiler.target>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
</properties>
<dependencies>
- <!-- provided -->
<dependency>
<groupId>org.apache.kylin</groupId>
- <artifactId>kylin-core-metadata</artifactId>
+ <artifactId>kylin-core-common</artifactId>
</dependency>
<dependency>
<groupId>io.kyligence.ke</groupId>
- <artifactId>kap-external-curator-test</artifactId>
+ <artifactId>kap-external-influxdb</artifactId>
</dependency>
<dependency>
- <groupId>org.apache.kylin</groupId>
- <artifactId>kylin-core-common</artifactId>
- <type>test-jar</type>
- <scope>test</scope>
+ <groupId>io.dropwizard.metrics</groupId>
+ <artifactId>metrics-core</artifactId>
</dependency>
<dependency>
- <groupId>org.apache.kylin</groupId>
- <artifactId>kylin-core-metadata</artifactId>
+ <groupId>io.dropwizard.metrics</groupId>
+ <artifactId>metrics-jvm</artifactId>
</dependency>
-
<dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
+ <groupId>io.dropwizard.metrics</groupId>
+ <artifactId>metrics-json</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>io.dropwizard.metrics</groupId>
+ <artifactId>metrics-jmx</artifactId>
</dependency>
+
<dependency>
- <groupId>org.apache.zookeeper</groupId>
- <artifactId>zookeeper</artifactId>
+ <groupId>commons-collections</groupId>
+ <artifactId>commons-collections</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>commons-lang</groupId>
+ <artifactId>commons-lang</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework.security</groupId>
+ <artifactId>spring-security-core</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
<scope>provided</scope>
- <exclusions>
- <exclusion>
- <groupId>org.junit.vintage</groupId>
- <artifactId>junit-vintage-engine</artifactId>
- </exclusion>
- </exclusions>
</dependency>
- <!--Env & Test-->
+ <dependency>
+ <groupId>org.apache.kylin</groupId>
+ <artifactId>kylin-core-common</artifactId>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework</groupId>
+ <artifactId>spring-test</artifactId>
+ <scope>test</scope>
+ </dependency>
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter-api</artifactId>
@@ -84,30 +89,41 @@
<scope>test</scope>
</dependency>
<dependency>
- <groupId>org.assertj</groupId>
- <artifactId>assertj-core</artifactId>
+ <groupId>org.junit-pioneer</groupId>
+ <artifactId>junit-pioneer</artifactId>
<scope>test</scope>
</dependency>
<dependency>
- <groupId>org.awaitility</groupId>
- <artifactId>awaitility</artifactId>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-core</artifactId>
<scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-core</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
- <groupId>org.mockito</groupId>
- <artifactId>mockito-core</artifactId>
+ <groupId>org.powermock</groupId>
+ <artifactId>powermock-module-junit4</artifactId>
<scope>test</scope>
</dependency>
<dependency>
- <groupId>org.mockito</groupId>
- <artifactId>mockito-inline</artifactId>
+ <groupId>org.powermock</groupId>
+ <artifactId>powermock-api-mockito2</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.assertj</groupId>
+ <artifactId>assertj-core</artifactId>
<scope>test</scope>
</dependency>
<dependency>
- <groupId>com.h2database</groupId>
- <artifactId>h2</artifactId>
+ <groupId>org.javassist</groupId>
+ <artifactId>javassist</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
-</project>
+</project>
\ No newline at end of file
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/metrics/MetricsCategory.java b/src/core-metrics/src/main/java/org/apache/kylin/common/metrics/MetricsCategory.java
similarity index 100%
rename from src/core-common/src/main/java/org/apache/kylin/common/metrics/MetricsCategory.java
rename to src/core-metrics/src/main/java/org/apache/kylin/common/metrics/MetricsCategory.java
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/metrics/MetricsConfig.java b/src/core-metrics/src/main/java/org/apache/kylin/common/metrics/MetricsConfig.java
similarity index 100%
rename from src/core-common/src/main/java/org/apache/kylin/common/metrics/MetricsConfig.java
rename to src/core-metrics/src/main/java/org/apache/kylin/common/metrics/MetricsConfig.java
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/metrics/MetricsController.java b/src/core-metrics/src/main/java/org/apache/kylin/common/metrics/MetricsController.java
similarity index 100%
rename from src/core-common/src/main/java/org/apache/kylin/common/metrics/MetricsController.java
rename to src/core-metrics/src/main/java/org/apache/kylin/common/metrics/MetricsController.java
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/metrics/MetricsGroup.java b/src/core-metrics/src/main/java/org/apache/kylin/common/metrics/MetricsGroup.java
similarity index 100%
rename from src/core-common/src/main/java/org/apache/kylin/common/metrics/MetricsGroup.java
rename to src/core-metrics/src/main/java/org/apache/kylin/common/metrics/MetricsGroup.java
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/metrics/MetricsInfluxdbReporter.java b/src/core-metrics/src/main/java/org/apache/kylin/common/metrics/MetricsInfluxdbReporter.java
similarity index 100%
rename from src/core-common/src/main/java/org/apache/kylin/common/metrics/MetricsInfluxdbReporter.java
rename to src/core-metrics/src/main/java/org/apache/kylin/common/metrics/MetricsInfluxdbReporter.java
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/metrics/MetricsName.java b/src/core-metrics/src/main/java/org/apache/kylin/common/metrics/MetricsName.java
similarity index 100%
rename from src/core-common/src/main/java/org/apache/kylin/common/metrics/MetricsName.java
rename to src/core-metrics/src/main/java/org/apache/kylin/common/metrics/MetricsName.java
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/metrics/MetricsObject.java b/src/core-metrics/src/main/java/org/apache/kylin/common/metrics/MetricsObject.java
similarity index 100%
rename from src/core-common/src/main/java/org/apache/kylin/common/metrics/MetricsObject.java
rename to src/core-metrics/src/main/java/org/apache/kylin/common/metrics/MetricsObject.java
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/metrics/MetricsObjectType.java b/src/core-metrics/src/main/java/org/apache/kylin/common/metrics/MetricsObjectType.java
similarity index 100%
rename from src/core-common/src/main/java/org/apache/kylin/common/metrics/MetricsObjectType.java
rename to src/core-metrics/src/main/java/org/apache/kylin/common/metrics/MetricsObjectType.java
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/metrics/MetricsReporter.java b/src/core-metrics/src/main/java/org/apache/kylin/common/metrics/MetricsReporter.java
similarity index 100%
rename from src/core-common/src/main/java/org/apache/kylin/common/metrics/MetricsReporter.java
rename to src/core-metrics/src/main/java/org/apache/kylin/common/metrics/MetricsReporter.java
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/metrics/MetricsTag.java b/src/core-metrics/src/main/java/org/apache/kylin/common/metrics/MetricsTag.java
similarity index 100%
rename from src/core-common/src/main/java/org/apache/kylin/common/metrics/MetricsTag.java
rename to src/core-metrics/src/main/java/org/apache/kylin/common/metrics/MetricsTag.java
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/metrics/context/ClusterContext.java b/src/core-metrics/src/main/java/org/apache/kylin/common/metrics/context/ClusterContext.java
similarity index 100%
rename from src/core-common/src/main/java/org/apache/kylin/common/metrics/context/ClusterContext.java
rename to src/core-metrics/src/main/java/org/apache/kylin/common/metrics/context/ClusterContext.java
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/metrics/context/ClusterInfo.java b/src/core-metrics/src/main/java/org/apache/kylin/common/metrics/context/ClusterInfo.java
similarity index 100%
rename from src/core-common/src/main/java/org/apache/kylin/common/metrics/context/ClusterInfo.java
rename to src/core-metrics/src/main/java/org/apache/kylin/common/metrics/context/ClusterInfo.java
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/metrics/gauges/QueryRatioGauge.java b/src/core-metrics/src/main/java/org/apache/kylin/common/metrics/gauges/QueryRatioGauge.java
similarity index 100%
rename from src/core-common/src/main/java/org/apache/kylin/common/metrics/gauges/QueryRatioGauge.java
rename to src/core-metrics/src/main/java/org/apache/kylin/common/metrics/gauges/QueryRatioGauge.java
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/metrics/prometheus/PrometheusMetrics.java b/src/core-metrics/src/main/java/org/apache/kylin/common/metrics/prometheus/PrometheusMetrics.java
similarity index 100%
rename from src/core-common/src/main/java/org/apache/kylin/common/metrics/prometheus/PrometheusMetrics.java
rename to src/core-metrics/src/main/java/org/apache/kylin/common/metrics/prometheus/PrometheusMetrics.java
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/metrics/reporter/InfluxdbReporter.java b/src/core-metrics/src/main/java/org/apache/kylin/common/metrics/reporter/InfluxdbReporter.java
similarity index 100%
rename from src/core-common/src/main/java/org/apache/kylin/common/metrics/reporter/InfluxdbReporter.java
rename to src/core-metrics/src/main/java/org/apache/kylin/common/metrics/reporter/InfluxdbReporter.java
index 965a183567..5686c85f97 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/metrics/reporter/InfluxdbReporter.java
+++ b/src/core-metrics/src/main/java/org/apache/kylin/common/metrics/reporter/InfluxdbReporter.java
@@ -36,12 +36,12 @@ import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.TimeUnit;
import java.util.function.Function;
-import org.apache.kylin.common.util.Pair;
import org.apache.kylin.common.metrics.MetricsCategory;
import org.apache.kylin.common.metrics.MetricsGroup;
import org.apache.kylin.common.metrics.MetricsName;
import org.apache.kylin.common.metrics.service.InfluxDBInstance;
import org.apache.kylin.common.util.AddressUtil;
+import org.apache.kylin.common.util.Pair;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/metrics/reporter/ServerModeMetricFilter.java b/src/core-metrics/src/main/java/org/apache/kylin/common/metrics/reporter/ServerModeMetricFilter.java
similarity index 100%
rename from src/core-common/src/main/java/org/apache/kylin/common/metrics/reporter/ServerModeMetricFilter.java
rename to src/core-metrics/src/main/java/org/apache/kylin/common/metrics/reporter/ServerModeMetricFilter.java
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/metrics/service/InfluxDBInstance.java b/src/core-metrics/src/main/java/org/apache/kylin/common/metrics/service/InfluxDBInstance.java
similarity index 100%
rename from src/core-common/src/main/java/org/apache/kylin/common/metrics/service/InfluxDBInstance.java
rename to src/core-metrics/src/main/java/org/apache/kylin/common/metrics/service/InfluxDBInstance.java
index fc821e01e3..3f8e96f31b 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/metrics/service/InfluxDBInstance.java
+++ b/src/core-metrics/src/main/java/org/apache/kylin/common/metrics/service/InfluxDBInstance.java
@@ -26,10 +26,10 @@ import java.util.concurrent.TimeUnit;
import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KapConfig;
import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.ExecutorServiceUtil;
-import org.apache.kylin.common.util.NamedThreadFactory;
import org.apache.kylin.common.metrics.MetricsGroup;
+import org.apache.kylin.common.util.ExecutorServiceUtil;
import org.apache.kylin.common.util.InfluxDBUtils;
+import org.apache.kylin.common.util.NamedThreadFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/metrics/service/JobStatusMonitorMetric.java b/src/core-metrics/src/main/java/org/apache/kylin/common/metrics/service/JobStatusMonitorMetric.java
similarity index 100%
rename from src/core-common/src/main/java/org/apache/kylin/common/metrics/service/JobStatusMonitorMetric.java
rename to src/core-metrics/src/main/java/org/apache/kylin/common/metrics/service/JobStatusMonitorMetric.java
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/metrics/service/MonitorDao.java b/src/core-metrics/src/main/java/org/apache/kylin/common/metrics/service/MonitorDao.java
similarity index 100%
rename from src/core-common/src/main/java/org/apache/kylin/common/metrics/service/MonitorDao.java
rename to src/core-metrics/src/main/java/org/apache/kylin/common/metrics/service/MonitorDao.java
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/metrics/service/MonitorMetric.java b/src/core-metrics/src/main/java/org/apache/kylin/common/metrics/service/MonitorMetric.java
similarity index 100%
rename from src/core-common/src/main/java/org/apache/kylin/common/metrics/service/MonitorMetric.java
rename to src/core-metrics/src/main/java/org/apache/kylin/common/metrics/service/MonitorMetric.java
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/metrics/service/MonitorMetricOperation.java b/src/core-metrics/src/main/java/org/apache/kylin/common/metrics/service/MonitorMetricOperation.java
similarity index 100%
rename from src/core-common/src/main/java/org/apache/kylin/common/metrics/service/MonitorMetricOperation.java
rename to src/core-metrics/src/main/java/org/apache/kylin/common/metrics/service/MonitorMetricOperation.java
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/metrics/service/QueryMonitorMetric.java b/src/core-metrics/src/main/java/org/apache/kylin/common/metrics/service/QueryMonitorMetric.java
similarity index 100%
rename from src/core-common/src/main/java/org/apache/kylin/common/metrics/service/QueryMonitorMetric.java
rename to src/core-metrics/src/main/java/org/apache/kylin/common/metrics/service/QueryMonitorMetric.java
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/util/InfluxDBUtils.java b/src/core-metrics/src/main/java/org/apache/kylin/common/util/InfluxDBUtils.java
similarity index 100%
rename from src/core-common/src/main/java/org/apache/kylin/common/util/InfluxDBUtils.java
rename to src/core-metrics/src/main/java/org/apache/kylin/common/util/InfluxDBUtils.java
diff --git a/src/core-common/src/test/java/org/apache/kylin/common/metric/InfluxDBInstanceTest.java b/src/core-metrics/src/test/java/org/apache/kylin/common/metric/InfluxDBInstanceTest.java
similarity index 100%
rename from src/core-common/src/test/java/org/apache/kylin/common/metric/InfluxDBInstanceTest.java
rename to src/core-metrics/src/test/java/org/apache/kylin/common/metric/InfluxDBInstanceTest.java
diff --git a/src/core-common/src/test/java/org/apache/kylin/common/metric/InfluxMetricsTest.java b/src/core-metrics/src/test/java/org/apache/kylin/common/metric/InfluxMetricsTest.java
similarity index 100%
rename from src/core-common/src/test/java/org/apache/kylin/common/metric/InfluxMetricsTest.java
rename to src/core-metrics/src/test/java/org/apache/kylin/common/metric/InfluxMetricsTest.java
diff --git a/src/core-common/src/test/java/org/apache/kylin/common/metric/MetricsControllerTest.java b/src/core-metrics/src/test/java/org/apache/kylin/common/metric/MetricsControllerTest.java
similarity index 89%
rename from src/core-common/src/test/java/org/apache/kylin/common/metric/MetricsControllerTest.java
rename to src/core-metrics/src/test/java/org/apache/kylin/common/metric/MetricsControllerTest.java
index 64a1ba1a58..5786d4ed12 100644
--- a/src/core-common/src/test/java/org/apache/kylin/common/metric/MetricsControllerTest.java
+++ b/src/core-metrics/src/test/java/org/apache/kylin/common/metric/MetricsControllerTest.java
@@ -42,18 +42,18 @@ import org.springframework.security.core.context.SecurityContextHolder;
import io.kyligence.kap.shaded.influxdb.org.influxdb.InfluxDB;
@RunWith(PowerMockRunner.class)
-@PowerMockIgnore({"javax.net.ssl.*", "javax.management.*", "org.apache.hadoop.*", "javax.security.*", "javax.crypto.*", "javax.script.*"})
-@PrepareForTest({InfluxDBInstance.class, InfluxDBUtils.class, UserGroupInformation.class})
-public class MetricsControllerTest extends NLocalFileMetadataTestCase{
+@PowerMockIgnore({ "javax.net.ssl.*", "javax.management.*", "org.apache.hadoop.*", "javax.security.*", "javax.crypto.*",
+ "javax.script.*" })
+@PrepareForTest({ InfluxDBInstance.class, InfluxDBUtils.class, UserGroupInformation.class })
+public class MetricsControllerTest extends NLocalFileMetadataTestCase {
public final static String ROLE_ADMIN = "ROLE_ADMIN";
private final Authentication authentication = new TestingAuthenticationToken("ADMIN", "ADMIN", ROLE_ADMIN);
-
@Before
public void setUp() throws Exception {
PowerMockito.mockStatic(UserGroupInformation.class);
PowerMockito.mockStatic(InfluxDBUtils.class);
-
+
UserGroupInformation userGroupInformation = Mockito.mock(UserGroupInformation.class);
PowerMockito.when(UserGroupInformation.getCurrentUser()).thenAnswer(invocation -> userGroupInformation);
overwriteSystemProp("HADOOP_USER_NAME", "root");
@@ -66,14 +66,14 @@ public class MetricsControllerTest extends NLocalFileMetadataTestCase{
cleanupTestMetadata();
}
-
@Test
public void initTest() throws Exception {
KapConfig config = KapConfig.wrap(KylinConfig.getInstanceFromEnv());
InfluxDB influxDB = Mockito.mock(InfluxDB.class);
- PowerMockito.doAnswer(invocationOnMock -> influxDB).when(InfluxDBUtils.class, "getInfluxDBInstance", Mockito.anyString(), Mockito.anyString(),
- Mockito.anyString(), Mockito.anyBoolean(), Mockito.anyBoolean());
+ PowerMockito.doAnswer(invocationOnMock -> influxDB).when(InfluxDBUtils.class, "getInfluxDBInstance",
+ Mockito.anyString(), Mockito.anyString(), Mockito.anyString(), Mockito.anyBoolean(),
+ Mockito.anyBoolean());
Mockito.when(influxDB.databaseExists(Mockito.anyString())).thenReturn(true);
MetricsController.init(config);
diff --git a/src/core-common/src/test/java/org/apache/kylin/common/metric/MonitorDaoTest.java b/src/core-metrics/src/test/java/org/apache/kylin/common/metric/MonitorDaoTest.java
similarity index 100%
rename from src/core-common/src/test/java/org/apache/kylin/common/metric/MonitorDaoTest.java
rename to src/core-metrics/src/test/java/org/apache/kylin/common/metric/MonitorDaoTest.java
diff --git a/src/data-loading-server/src/main/java/org/apache/kylin/rest/controller/BaseController.java b/src/data-loading-server/src/main/java/org/apache/kylin/rest/controller/BaseController.java
index a9eadeff1d..b5a2cec688 100644
--- a/src/data-loading-server/src/main/java/org/apache/kylin/rest/controller/BaseController.java
+++ b/src/data-loading-server/src/main/java/org/apache/kylin/rest/controller/BaseController.java
@@ -68,7 +68,6 @@ import org.apache.kylin.common.msg.MsgPicker;
import org.apache.kylin.common.persistence.transaction.TransactionException;
import org.apache.kylin.common.util.DateFormat;
import org.apache.kylin.common.util.JsonUtil;
-import org.apache.kylin.common.util.Unsafe;
import org.apache.kylin.job.dao.ExecutablePO;
import org.apache.kylin.metadata.project.NProjectManager;
import org.apache.kylin.metadata.project.ProjectInstance;
@@ -151,7 +150,7 @@ public class BaseController {
if (kylinException != null) {
cause = kylinException;
}
- return new ErrorResponse(Unsafe.getUrlFromHttpServletRequest(req), cause);
+ return new ErrorResponse(req.getRequestURL().toString(), cause);
}
@ResponseStatus(HttpStatus.FORBIDDEN)
@@ -159,7 +158,7 @@ public class BaseController {
@ResponseBody
ErrorResponse handleForbidden(HttpServletRequest req, Exception ex) {
getLogger().error("", ex);
- return new ErrorResponse(Unsafe.getUrlFromHttpServletRequest(req), ex);
+ return new ErrorResponse(req.getRequestURL().toString(), ex);
}
@ResponseStatus(HttpStatus.NOT_FOUND)
@@ -167,7 +166,7 @@ public class BaseController {
@ResponseBody
ErrorResponse handleNotFound(HttpServletRequest req, Exception ex) {
getLogger().error("", ex);
- return new ErrorResponse(Unsafe.getUrlFromHttpServletRequest(req), ex);
+ return new ErrorResponse(req.getRequestURL().toString(), ex);
}
@ResponseStatus(HttpStatus.INTERNAL_SERVER_ERROR)
@@ -191,7 +190,7 @@ public class BaseController {
ErrorResponse handleAccessDenied(HttpServletRequest req, Throwable ex) {
getLogger().error("", ex);
KylinException e = new KylinException(ACCESS_DENIED, MsgPicker.getMsg().getAccessDeny());
- return new ErrorResponse(Unsafe.getUrlFromHttpServletRequest(req), e);
+ return new ErrorResponse(req.getRequestURL().toString(), e);
}
@ResponseStatus(HttpStatus.BAD_REQUEST)
@@ -200,7 +199,7 @@ public class BaseController {
ErrorResponse handleInvalidRequestParam(HttpServletRequest req, Throwable ex) {
KylinException e = new KylinException(INVALID_PARAMETER, ex);
getLogger().error("", e);
- return new ErrorResponse(Unsafe.getUrlFromHttpServletRequest(req), e);
+ return new ErrorResponse(req.getRequestURL().toString(), e);
}
@ResponseStatus(HttpStatus.BAD_REQUEST)
@@ -210,7 +209,7 @@ public class BaseController {
String parameterName = ex.getParameterName();
KylinException e = new KylinException(REQUEST_PARAMETER_EMPTY_OR_VALUE_EMPTY, parameterName);
getLogger().error("", e);
- return new ErrorResponse(Unsafe.getUrlFromHttpServletRequest(req), e);
+ return new ErrorResponse(req.getRequestURL().toString(), e);
}
@ResponseStatus(HttpStatus.INTERNAL_SERVER_ERROR)
@@ -219,14 +218,14 @@ public class BaseController {
ErrorResponse handleErrorCode(HttpServletRequest req, Throwable ex) {
getLogger().error("", ex);
KylinException cause = (KylinException) ex;
- return new ErrorResponse(Unsafe.getUrlFromHttpServletRequest(req), cause);
+ return new ErrorResponse(req.getRequestURL().toString(), cause);
}
@ResponseStatus(HttpStatus.BAD_REQUEST)
@ExceptionHandler(MethodArgumentNotValidException.class)
@ResponseBody
ErrorResponse handleInvalidArgument(HttpServletRequest request, MethodArgumentNotValidException ex) {
- val response = new ErrorResponse(Unsafe.getUrlFromHttpServletRequest(request), ex);
+ val response = new ErrorResponse(request.getRequestURL().toString(), ex);
val target = ex.getBindingResult().getTarget();
if (target instanceof Validation) {
response.setMsg(((Validation) target).getErrorMessage(ex.getBindingResult().getFieldErrors()));
@@ -244,7 +243,7 @@ public class BaseController {
ErrorResponse handleUnauthorized(HttpServletRequest req, Throwable ex) {
KylinException e = new KylinException(USER_UNAUTHORIZED, ex);
getLogger().error("", e);
- return new ErrorResponse(Unsafe.getUrlFromHttpServletRequest(req), ex);
+ return new ErrorResponse(req.getRequestURL().toString(), ex);
}
protected void checkRequiredArg(String fieldName, Object fieldValue) {
diff --git a/src/data-loading-service/src/main/java/org/apache/kylin/rest/filter/SegmentsRequestFilter.java b/src/data-loading-service/src/main/java/org/apache/kylin/rest/filter/SegmentsRequestFilter.java
index 42477bede6..63256b746b 100644
--- a/src/data-loading-service/src/main/java/org/apache/kylin/rest/filter/SegmentsRequestFilter.java
+++ b/src/data-loading-service/src/main/java/org/apache/kylin/rest/filter/SegmentsRequestFilter.java
@@ -42,7 +42,6 @@ import org.apache.kylin.common.exception.ErrorCode;
import org.apache.kylin.common.exception.KylinException;
import org.apache.kylin.common.msg.MsgPicker;
import org.apache.kylin.common.util.JsonUtil;
-import org.apache.kylin.common.util.Unsafe;
import org.apache.kylin.rest.response.ErrorResponse;
import org.springframework.core.annotation.Order;
import org.springframework.http.HttpHeaders;
@@ -84,7 +83,7 @@ public class SegmentsRequestFilter implements Filter {
MsgPicker.setMsg(httpServletRequest.getHeader(HttpHeaders.ACCEPT_LANGUAGE));
ErrorCode.setMsg(httpServletRequest.getHeader(HttpHeaders.ACCEPT_LANGUAGE));
- ErrorResponse errorResponse = new ErrorResponse(Unsafe.getUrlFromHttpServletRequest(httpServletRequest),
+ ErrorResponse errorResponse = new ErrorResponse(httpServletRequest.getRequestURL().toString(),
e);
byte[] responseBody = JsonUtil.writeValueAsBytes(errorResponse);
diff --git a/src/datasource-sdk/pom.xml b/src/datasource-sdk/pom.xml
index 27bcc81db4..992fac1baa 100644
--- a/src/datasource-sdk/pom.xml
+++ b/src/datasource-sdk/pom.xml
@@ -14,11 +14,6 @@
<name>Kylin - Datasource SDK</name>
<dependencies>
- <!-- provided -->
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- </dependency>
<dependency>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin-core-metadata</artifactId>
@@ -28,10 +23,6 @@
<artifactId>jackson-dataformat-xml</artifactId>
</dependency>
- <dependency>
- <groupId>org.apache.calcite</groupId>
- <artifactId>calcite-linq4j</artifactId>
- </dependency>
<!--Test and Environment-->
<dependency>
<groupId>org.apache.kylin</groupId>
@@ -40,14 +31,8 @@
<scope>test</scope>
</dependency>
<dependency>
- <groupId>org.apache.kylin</groupId>
- <artifactId>kylin-core-metadata</artifactId>
- <type>test-jar</type>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.kylin</groupId>
- <artifactId>kylin-source-hive</artifactId>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
<scope>test</scope>
</dependency>
<dependency>
@@ -56,13 +41,13 @@
<scope>test</scope>
</dependency>
<dependency>
- <groupId>com.h2database</groupId>
- <artifactId>h2</artifactId>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-core</artifactId>
<scope>test</scope>
</dependency>
<dependency>
- <groupId>org.mockito</groupId>
- <artifactId>mockito-core</artifactId>
+ <groupId>com.h2database</groupId>
+ <artifactId>h2</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
diff --git a/src/datasource-service/pom.xml b/src/datasource-service/pom.xml
index 69e2d8427b..acd5eacfa9 100644
--- a/src/datasource-service/pom.xml
+++ b/src/datasource-service/pom.xml
@@ -80,6 +80,10 @@
<artifactId>spark-sql_2.12</artifactId>
<scope>compile</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.kylin</groupId>
+ <artifactId>distributed-lock-ext</artifactId>
+ </dependency>
<!-- test -->
<dependency>
diff --git a/src/core-job/pom.xml b/src/distributed-lock-ext/pom.xml
similarity index 51%
copy from src/core-job/pom.xml
copy to src/distributed-lock-ext/pom.xml
index bbdbb0cf2f..3df312fd32 100644
--- a/src/core-job/pom.xml
+++ b/src/distributed-lock-ext/pom.xml
@@ -1,91 +1,79 @@
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
+<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
- <artifactId>kylin-core-job</artifactId>
- <packaging>jar</packaging>
- <name>Kylin - Core Job</name>
- <url>http://kylin.apache.org</url>
- <description>Kylin Core Job</description>
-
<parent>
- <groupId>org.apache.kylin</groupId>
<artifactId>kylin</artifactId>
+ <groupId>org.apache.kylin</groupId>
<version>5.0.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>distributed-lock-ext</artifactId>
+ <name>Kylin - Distributed Lock Extension</name>
<properties>
- <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+ <maven.compiler.source>8</maven.compiler.source>
+ <maven.compiler.target>8</maven.compiler.target>
</properties>
<dependencies>
- <!-- provided -->
<dependency>
<groupId>org.apache.kylin</groupId>
- <artifactId>kylin-core-metadata</artifactId>
+ <artifactId>kylin-core-common</artifactId>
</dependency>
<dependency>
<groupId>io.kyligence.ke</groupId>
- <artifactId>kap-external-curator-test</artifactId>
+ <artifactId>kap-external-curator</artifactId>
</dependency>
<dependency>
- <groupId>org.apache.kylin</groupId>
- <artifactId>kylin-core-common</artifactId>
- <type>test-jar</type>
- <scope>test</scope>
+ <groupId>org.springframework.integration</groupId>
+ <artifactId>spring-integration-jdbc</artifactId>
</dependency>
<dependency>
- <groupId>org.apache.kylin</groupId>
- <artifactId>kylin-core-metadata</artifactId>
+ <groupId>org.mybatis</groupId>
+ <artifactId>mybatis</artifactId>
</dependency>
-
<dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
+ <groupId>org.mybatis.dynamic-sql</groupId>
+ <artifactId>mybatis-dynamic-sql</artifactId>
</dependency>
<dependency>
- <groupId>org.apache.zookeeper</groupId>
- <artifactId>zookeeper</artifactId>
+ <groupId>org.projectlombok</groupId>
+ <artifactId>lombok</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-dbcp2</artifactId>
<scope>provided</scope>
- <exclusions>
- <exclusion>
- <groupId>org.junit.vintage</groupId>
- <artifactId>junit-vintage-engine</artifactId>
- </exclusion>
- </exclusions>
</dependency>
- <!--Env & Test-->
<dependency>
- <groupId>org.junit.jupiter</groupId>
- <artifactId>junit-jupiter-api</artifactId>
+ <groupId>org.apache.kylin</groupId>
+ <artifactId>kylin-core-common</artifactId>
+ <type>test-jar</type>
<scope>test</scope>
</dependency>
<dependency>
- <groupId>org.junit.vintage</groupId>
- <artifactId>junit-vintage-engine</artifactId>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.h2database</groupId>
+ <artifactId>h2</artifactId>
<scope>test</scope>
</dependency>
<dependency>
- <groupId>org.assertj</groupId>
- <artifactId>assertj-core</artifactId>
+ <groupId>org.springframework</groupId>
+ <artifactId>spring-test</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.curator</groupId>
+ <artifactId>curator-test</artifactId>
<scope>test</scope>
</dependency>
<dependency>
@@ -94,20 +82,21 @@
<scope>test</scope>
</dependency>
<dependency>
- <groupId>org.mockito</groupId>
- <artifactId>mockito-core</artifactId>
+ <groupId>org.junit.jupiter</groupId>
+ <artifactId>junit-jupiter-api</artifactId>
<scope>test</scope>
</dependency>
<dependency>
- <groupId>org.mockito</groupId>
- <artifactId>mockito-inline</artifactId>
+ <groupId>org.junit.vintage</groupId>
+ <artifactId>junit-vintage-engine</artifactId>
<scope>test</scope>
</dependency>
<dependency>
- <groupId>com.h2database</groupId>
- <artifactId>h2</artifactId>
+ <groupId>org.junit-pioneer</groupId>
+ <artifactId>junit-pioneer</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
-</project>
+
+</project>
\ No newline at end of file
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/lock/curator/CuratorDistributedLock.java b/src/distributed-lock-ext/src/main/java/org/apache/kylin/common/lock/curator/CuratorDistributedLock.java
similarity index 100%
rename from src/core-common/src/main/java/org/apache/kylin/common/lock/curator/CuratorDistributedLock.java
rename to src/distributed-lock-ext/src/main/java/org/apache/kylin/common/lock/curator/CuratorDistributedLock.java
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/lock/curator/CuratorDistributedLockFactory.java b/src/distributed-lock-ext/src/main/java/org/apache/kylin/common/lock/curator/CuratorDistributedLockFactory.java
similarity index 100%
rename from src/core-common/src/main/java/org/apache/kylin/common/lock/curator/CuratorDistributedLockFactory.java
rename to src/distributed-lock-ext/src/main/java/org/apache/kylin/common/lock/curator/CuratorDistributedLockFactory.java
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/lock/jdbc/JdbcDistributedLockFactory.java b/src/distributed-lock-ext/src/main/java/org/apache/kylin/common/lock/jdbc/JdbcDistributedLockFactory.java
similarity index 100%
rename from src/core-common/src/main/java/org/apache/kylin/common/lock/jdbc/JdbcDistributedLockFactory.java
rename to src/distributed-lock-ext/src/main/java/org/apache/kylin/common/lock/jdbc/JdbcDistributedLockFactory.java
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/lock/jdbc/JdbcDistributedLockUtil.java b/src/distributed-lock-ext/src/main/java/org/apache/kylin/common/lock/jdbc/JdbcDistributedLockUtil.java
similarity index 99%
rename from src/core-common/src/main/java/org/apache/kylin/common/lock/jdbc/JdbcDistributedLockUtil.java
rename to src/distributed-lock-ext/src/main/java/org/apache/kylin/common/lock/jdbc/JdbcDistributedLockUtil.java
index be93e5da19..edd7eddb03 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/lock/jdbc/JdbcDistributedLockUtil.java
+++ b/src/distributed-lock-ext/src/main/java/org/apache/kylin/common/lock/jdbc/JdbcDistributedLockUtil.java
@@ -18,17 +18,6 @@
package org.apache.kylin.common.lock.jdbc;
-import lombok.extern.slf4j.Slf4j;
-import lombok.val;
-import org.apache.commons.dbcp2.BasicDataSource;
-import org.apache.ibatis.jdbc.ScriptRunner;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.StorageURL;
-import org.apache.kylin.common.logging.LogOutputStream;
-import org.apache.kylin.common.persistence.metadata.JdbcDataSource;
-import org.apache.kylin.common.persistence.metadata.jdbc.JdbcUtil;
-
-import javax.sql.DataSource;
import java.io.ByteArrayInputStream;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
@@ -38,6 +27,19 @@ import java.sql.Connection;
import java.util.Locale;
import java.util.Properties;
+import javax.sql.DataSource;
+
+import org.apache.commons.dbcp2.BasicDataSource;
+import org.apache.ibatis.jdbc.ScriptRunner;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.StorageURL;
+import org.apache.kylin.common.logging.LogOutputStream;
+import org.apache.kylin.common.persistence.metadata.JdbcDataSource;
+import org.apache.kylin.common.persistence.metadata.jdbc.JdbcUtil;
+
+import lombok.val;
+import lombok.extern.slf4j.Slf4j;
+
@Slf4j
public class JdbcDistributedLockUtil {
private static final Charset DEFAULT_CHARSET = Charset.defaultCharset();
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/util/ZKUtil.java b/src/distributed-lock-ext/src/main/java/org/apache/kylin/common/util/ZKUtil.java
similarity index 96%
rename from src/core-common/src/main/java/org/apache/kylin/common/util/ZKUtil.java
rename to src/distributed-lock-ext/src/main/java/org/apache/kylin/common/util/ZKUtil.java
index dbbeb9277a..6ab6e860ab 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/util/ZKUtil.java
+++ b/src/distributed-lock-ext/src/main/java/org/apache/kylin/common/util/ZKUtil.java
@@ -18,9 +18,6 @@
package org.apache.kylin.common.util;
-import static org.apache.kylin.common.ZookeeperConfig.geZKClientConnectionTimeoutMs;
-import static org.apache.kylin.common.ZookeeperConfig.geZKClientSessionTimeoutMs;
-
import java.io.File;
import java.io.IOException;
import java.util.concurrent.Callable;
@@ -28,6 +25,7 @@ import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.ZookeeperConfig;
import org.apache.zookeeper.KeeperException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -177,8 +175,8 @@ public class ZKUtil {
if (!Strings.isNullOrEmpty(zkRoot)) {
zkString += zkRoot;
}
- int sessionTimeout = geZKClientSessionTimeoutMs();
- int connectionTimeout = geZKClientConnectionTimeoutMs();
+ int sessionTimeout = ZookeeperConfig.geZKClientSessionTimeoutMs();
+ int connectionTimeout = ZookeeperConfig.geZKClientConnectionTimeoutMs();
return CuratorFrameworkFactory.newClient(zkString, sessionTimeout, connectionTimeout, retryPolicy);
}
diff --git a/src/core-common/src/test/java/org/apache/kylin/common/lock/DistributedLockFactoryTest.java b/src/distributed-lock-ext/src/test/java/org/apache/kylin/common/lock/DistributedLockFactoryTest.java
similarity index 98%
rename from src/core-common/src/test/java/org/apache/kylin/common/lock/DistributedLockFactoryTest.java
rename to src/distributed-lock-ext/src/test/java/org/apache/kylin/common/lock/DistributedLockFactoryTest.java
index 2b774bcd41..ea90eed3ba 100644
--- a/src/core-common/src/test/java/org/apache/kylin/common/lock/DistributedLockFactoryTest.java
+++ b/src/distributed-lock-ext/src/test/java/org/apache/kylin/common/lock/DistributedLockFactoryTest.java
@@ -18,8 +18,8 @@
package org.apache.kylin.common.lock;
-import lombok.extern.slf4j.Slf4j;
-import org.junit.Assert;
+import static org.apache.kylin.common.util.TestUtils.getTestConfig;
+import static org.awaitility.Awaitility.await;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
@@ -27,8 +27,9 @@ import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.Lock;
-import static org.apache.kylin.common.util.TestUtils.getTestConfig;
-import static org.awaitility.Awaitility.await;
+import org.junit.Assert;
+
+import lombok.extern.slf4j.Slf4j;
@Slf4j
public class DistributedLockFactoryTest {
@@ -39,7 +40,7 @@ public class DistributedLockFactoryTest {
final CountDownLatch tasks = new CountDownLatch(threadNum);
- int[] count = new int[] {0};
+ int[] count = new int[] { 0 };
for (int i = 0; i < threadNum; i++) {
executorService.submit(new DirtyReadTest(key, times, tasks, count));
diff --git a/src/core-common/src/test/java/org/apache/kylin/common/lock/curator/CuratorDistributedLockFactoryTest.java b/src/distributed-lock-ext/src/test/java/org/apache/kylin/common/lock/curator/CuratorDistributedLockFactoryTest.java
similarity index 73%
rename from src/core-common/src/test/java/org/apache/kylin/common/lock/curator/CuratorDistributedLockFactoryTest.java
rename to src/distributed-lock-ext/src/test/java/org/apache/kylin/common/lock/curator/CuratorDistributedLockFactoryTest.java
index ea3692a8d0..cec38b22ca 100644
--- a/src/core-common/src/test/java/org/apache/kylin/common/lock/curator/CuratorDistributedLockFactoryTest.java
+++ b/src/distributed-lock-ext/src/test/java/org/apache/kylin/common/lock/curator/CuratorDistributedLockFactoryTest.java
@@ -18,9 +18,6 @@
package org.apache.kylin.common.lock.curator;
-import static org.apache.kylin.common.util.TestUtils.getTestConfig;
-import static org.awaitility.Awaitility.await;
-
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
@@ -29,7 +26,9 @@ import java.util.concurrent.locks.Lock;
import org.apache.curator.test.TestingServer;
import org.apache.kylin.common.lock.DistributedLockFactoryTest;
import org.apache.kylin.common.util.RandomUtil;
+import org.apache.kylin.common.util.TestUtils;
import org.apache.kylin.junit.annotation.MetadataInfo;
+import org.awaitility.Awaitility;
import org.junit.Assert;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
@@ -64,8 +63,9 @@ class CuratorDistributedLockFactoryTest extends DistributedLockFactoryTest {
public void testBasic() throws Exception {
String path = "/test/distributed_lock_factory_test/test_basic/" + RandomUtil.randomUUIDStr();
- getTestConfig().setProperty("kylin.env.zookeeper-connect-string", zkTestServer.getConnectString());
- CuratorDistributedLock lock = (CuratorDistributedLock) getTestConfig().getDistributedLockFactory().getLockForCurrentThread(path);
+ TestUtils.getTestConfig().setProperty("kylin.env.zookeeper-connect-string", zkTestServer.getConnectString());
+ CuratorDistributedLock lock = (CuratorDistributedLock) TestUtils.getTestConfig().getDistributedLockFactory()
+ .getLockForCurrentThread(path);
Assert.assertFalse(lock.isAcquiredInThisThread());
lock.lock();
@@ -81,14 +81,15 @@ class CuratorDistributedLockFactoryTest extends DistributedLockFactoryTest {
ExecutorService executorService = Executors.newFixedThreadPool(1);
- getTestConfig().setProperty("kylin.env.zookeeper-connect-string", zkTestServer.getConnectString());
- getTestConfig().setProperty("kap.env.zookeeper-max-retries", "1");
- getTestConfig().setProperty("kap.env.zookeeper-base-sleep-time", "1000");
+ TestUtils.getTestConfig().setProperty("kylin.env.zookeeper-connect-string", zkTestServer.getConnectString());
+ TestUtils.getTestConfig().setProperty("kap.env.zookeeper-max-retries", "1");
+ TestUtils.getTestConfig().setProperty("kap.env.zookeeper-base-sleep-time", "1000");
executorService.submit(() -> {
CuratorDistributedLock lock = null;
try {
- lock = (CuratorDistributedLock)getTestConfig().getDistributedLockFactory().getLockForCurrentThread(path);
+ lock = (CuratorDistributedLock) TestUtils.getTestConfig().getDistributedLockFactory()
+ .getLockForCurrentThread(path);
} catch (Exception e) {
e.printStackTrace();
}
@@ -103,7 +104,7 @@ class CuratorDistributedLockFactoryTest extends DistributedLockFactoryTest {
}
});
- await().atMost(5, TimeUnit.SECONDS).until(() -> locked);
+ Awaitility.await().atMost(5, TimeUnit.SECONDS).until(() -> locked);
Assert.assertFalse(isInterrupted);
locked = false;
@@ -111,15 +112,15 @@ class CuratorDistributedLockFactoryTest extends DistributedLockFactoryTest {
// zk for thread1 lost
// thread1 will be interrupted
- await().atMost(20, TimeUnit.SECONDS).until(() -> isInterrupted);
+ Awaitility.await().atMost(20, TimeUnit.SECONDS).until(() -> isInterrupted);
Assert.assertFalse(locked);
- getTestConfig().setProperty("kylin.env.zookeeper-connect-string", zkTestServer2.getConnectString());
+ TestUtils.getTestConfig().setProperty("kylin.env.zookeeper-connect-string", zkTestServer2.getConnectString());
executorService.submit(() -> {
Lock lock = null;
try {
- lock = getTestConfig().getDistributedLockFactory().getLockForCurrentThread(path);
+ lock = TestUtils.getTestConfig().getDistributedLockFactory().getLockForCurrentThread(path);
} catch (Exception e) {
e.printStackTrace();
}
@@ -130,7 +131,7 @@ class CuratorDistributedLockFactoryTest extends DistributedLockFactoryTest {
// thread1 released the lock
// thread2 will get the lock
- await().atMost(5, TimeUnit.SECONDS).until(() -> locked);
+ Awaitility.await().atMost(5, TimeUnit.SECONDS).until(() -> locked);
}
@Test
@@ -141,10 +142,10 @@ class CuratorDistributedLockFactoryTest extends DistributedLockFactoryTest {
ExecutorService executorService = Executors.newFixedThreadPool(1);
CuratorDistributedLockFactory lockFactory;
CuratorDistributedLock lock1;
- getTestConfig().setProperty("kylin.env.zookeeper-connect-string", zkTestServer.getConnectString());
- getTestConfig().setProperty("kap.env.zookeeper-max-retries", "1");
- getTestConfig().setProperty("kap.env.zookeeper-base-sleep-time", "1000");
- lockFactory = (CuratorDistributedLockFactory) getTestConfig().getDistributedLockFactory();
+ TestUtils.getTestConfig().setProperty("kylin.env.zookeeper-connect-string", zkTestServer.getConnectString());
+ TestUtils.getTestConfig().setProperty("kap.env.zookeeper-max-retries", "1");
+ TestUtils.getTestConfig().setProperty("kap.env.zookeeper-base-sleep-time", "1000");
+ lockFactory = (CuratorDistributedLockFactory) TestUtils.getTestConfig().getDistributedLockFactory();
lock1 = lockFactory.getLockForCurrentThread(path);
executorService.submit(() -> {
@@ -158,7 +159,7 @@ class CuratorDistributedLockFactoryTest extends DistributedLockFactoryTest {
}
});
- await().atMost(5, TimeUnit.SECONDS).until(() -> locked);
+ Awaitility.await().atMost(5, TimeUnit.SECONDS).until(() -> locked);
Assert.assertFalse(isInterrupted);
@@ -169,12 +170,12 @@ class CuratorDistributedLockFactoryTest extends DistributedLockFactoryTest {
// zk for thread1 suspended
// thread1 will be interrupted
- await().atMost(5, TimeUnit.SECONDS).until(() -> isInterrupted);
+ Awaitility.await().atMost(5, TimeUnit.SECONDS).until(() -> isInterrupted);
}
@Test
void testConcurrence() throws Exception {
- getTestConfig().setProperty("kylin.env.zookeeper-connect-string", zkTestServer.getConnectString());
+ TestUtils.getTestConfig().setProperty("kylin.env.zookeeper-connect-string", zkTestServer.getConnectString());
String key = "/test/distributed_lock_factory_test/test_concurrence/" + RandomUtil.randomUUIDStr();
super.testConcurrence(key, 10, 10);
}
diff --git a/src/core-common/src/test/java/org/apache/kylin/common/lock/jdbc/JdbcDistributedLockFactoryTest.java b/src/distributed-lock-ext/src/test/java/org/apache/kylin/common/lock/jdbc/JdbcDistributedLockFactoryTest.java
similarity index 91%
rename from src/core-common/src/test/java/org/apache/kylin/common/lock/jdbc/JdbcDistributedLockFactoryTest.java
rename to src/distributed-lock-ext/src/test/java/org/apache/kylin/common/lock/jdbc/JdbcDistributedLockFactoryTest.java
index 45e8b533a4..4e5af52c0b 100644
--- a/src/core-common/src/test/java/org/apache/kylin/common/lock/jdbc/JdbcDistributedLockFactoryTest.java
+++ b/src/distributed-lock-ext/src/test/java/org/apache/kylin/common/lock/jdbc/JdbcDistributedLockFactoryTest.java
@@ -18,26 +18,25 @@
package org.apache.kylin.common.lock.jdbc;
+import static org.apache.kylin.common.util.TestUtils.getTestConfig;
+
+import java.util.UUID;
+
import org.apache.kylin.common.lock.DistributedLockFactoryTest;
import org.apache.kylin.junit.annotation.MetadataInfo;
import org.apache.kylin.junit.annotation.OverwriteProp;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
-import java.util.UUID;
-
-import static org.apache.kylin.common.util.TestUtils.getTestConfig;
-
@MetadataInfo(onlyProps = true)
class JdbcDistributedLockFactoryTest extends DistributedLockFactoryTest {
@BeforeEach
public void setup() {
- getTestConfig().getDistributedLockFactory().initialize();
+ getTestConfig().getDistributedLockFactory().initialize();
}
- @OverwriteProp(key = "kylin.metadata.distributed-lock-impl",
- value = "org.apache.kylin.common.lock.jdbc.JdbcDistributedLockFactory")
+ @OverwriteProp(key = "kylin.metadata.distributed-lock-impl", value = "org.apache.kylin.common.lock.jdbc.JdbcDistributedLockFactory")
@Test
void testConcurrence() throws Exception {
getTestConfig().getDistributedLockFactory().initialize();
diff --git a/src/kylin-it/pom.xml b/src/kylin-it/pom.xml
index 19a5def3a6..abd2acc15f 100644
--- a/src/kylin-it/pom.xml
+++ b/src/kylin-it/pom.xml
@@ -11,6 +11,11 @@
<modelVersion>4.0.0</modelVersion>
<artifactId>kylin-it</artifactId>
+ <name>Kylin - Integration Test</name>
+
+ <properties>
+ <beanutils.version>1.9.4</beanutils.version>
+ </properties>
<dependencies>
<!--Springfox Swagger2 v2.9.2, swagger2markup v1.3.3. has specific requirement on guava and commons-beanutils -->
diff --git a/src/kylin-it/src/test/java/org/apache/kylin/streaming/StreamingMergeEntryTest.java b/src/kylin-it/src/test/java/org/apache/kylin/streaming/StreamingMergeEntryTest.java
index 59acd33a6f..59a6912cac 100644
--- a/src/kylin-it/src/test/java/org/apache/kylin/streaming/StreamingMergeEntryTest.java
+++ b/src/kylin-it/src/test/java/org/apache/kylin/streaming/StreamingMergeEntryTest.java
@@ -384,6 +384,8 @@ public class StreamingMergeEntryTest extends StreamingTestCase {
val entry = Mockito.spy(new StreamingMergeEntry());
val segments = new Segments<NDataSegment>();
+ val mgr = NDataflowManager.getInstance(getTestConfig(), PROJECT);
+ var dataflow = mgr.getDataflow(DATAFLOW_ID);
for (int i = 0; i < 3; i++) {
val start = LocalDate.parse("2000-01-01").plusMonths(i);
val end = start.plusMonths(1);
@@ -395,6 +397,7 @@ public class StreamingMergeEntryTest extends StreamingTestCase {
seg.setSegmentRange(segRange);
seg.setStatus(SegmentStatusEnum.READY);
seg.setAdditionalInfo(null);
+ seg.setDataflow(dataflow);
segments.add(seg);
}
@@ -407,6 +410,8 @@ public class StreamingMergeEntryTest extends StreamingTestCase {
val segments = new Segments<NDataSegment>();
val addInfo = new HashMap<String, String>();
+ val mgr = NDataflowManager.getInstance(getTestConfig(), PROJECT);
+ var dataflow = mgr.getDataflow(DATAFLOW_ID);
addInfo.put("abc", "2");
for (int i = 0; i < 3; i++) {
val start = LocalDate.parse("2000-01-01").plusMonths(i);
@@ -419,6 +424,7 @@ public class StreamingMergeEntryTest extends StreamingTestCase {
seg.setSegmentRange(segRange);
seg.setStatus(SegmentStatusEnum.READY);
seg.setAdditionalInfo(addInfo);
+ seg.setDataflow(dataflow);
segments.add(seg);
}
@@ -431,6 +437,8 @@ public class StreamingMergeEntryTest extends StreamingTestCase {
val segments = new Segments<NDataSegment>();
val addInfo = new HashMap<String, String>();
+ val mgr = NDataflowManager.getInstance(getTestConfig(), PROJECT);
+ var dataflow = mgr.getDataflow(DATAFLOW_ID);
addInfo.put(StreamingConstants.FILE_LAYER, "2");
for (int i = 0; i < 3; i++) {
val start = LocalDate.parse("2000-01-01").plusMonths(i);
@@ -443,6 +451,7 @@ public class StreamingMergeEntryTest extends StreamingTestCase {
seg.setSegmentRange(segRange);
seg.setStatus(SegmentStatusEnum.READY);
seg.setAdditionalInfo(addInfo);
+ seg.setDataflow(dataflow);
segments.add(seg);
}
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/util/AddTableNameSqlVisitor.java b/src/modeling-service/src/main/java/org/apache/kylin/common/util/AddTableNameSqlVisitor.java
similarity index 100%
rename from src/core-common/src/main/java/org/apache/kylin/common/util/AddTableNameSqlVisitor.java
rename to src/modeling-service/src/main/java/org/apache/kylin/common/util/AddTableNameSqlVisitor.java
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/util/ModifyTableNameSqlVisitor.java b/src/modeling-service/src/main/java/org/apache/kylin/common/util/ModifyTableNameSqlVisitor.java
similarity index 100%
rename from src/core-common/src/main/java/org/apache/kylin/common/util/ModifyTableNameSqlVisitor.java
rename to src/modeling-service/src/main/java/org/apache/kylin/common/util/ModifyTableNameSqlVisitor.java
diff --git a/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelSemanticHelper.java b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelSemanticHelper.java
index 4decdbb834..f6866f2708 100644
--- a/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelSemanticHelper.java
+++ b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelSemanticHelper.java
@@ -423,7 +423,7 @@ public class ModelSemanticHelper extends BasicService {
.forEach(x -> x.changeTableAlias(oldAliasName, newAliasName));
model.getAllMeasures().stream().filter(x -> !x.isTomb())
.forEach(x -> x.changeTableAlias(oldAliasName, newAliasName));
- model.getComputedColumnDescs().forEach(x -> x.changeTableAlias(oldAliasName, newAliasName));
+ model.getComputedColumnDescs().forEach(x -> changeTableAlias(x, oldAliasName, newAliasName));
String filterCondition = model.getFilterCondition();
if (StringUtils.isNotEmpty(filterCondition)) {
@@ -436,6 +436,13 @@ public class ModelSemanticHelper extends BasicService {
}
}
+ private void changeTableAlias(ComputedColumnDesc computedColumnDesc, String oldAlias, String newAlias) {
+ SqlVisitor<Object> modifyAlias = new ModifyTableNameSqlVisitor(oldAlias, newAlias);
+ SqlNode sqlNode = CalciteParser.getExpNode(computedColumnDesc.getExpression());
+ sqlNode.accept(modifyAlias);
+ computedColumnDesc.setExpression(sqlNode.toSqlString(HiveSqlDialect.DEFAULT).toString());
+ }
+
private Map<String, String> getAliasTransformMap(NDataModel originModel, NDataModel expectModel) {
Map<String, String> matchAlias = Maps.newHashMap();
boolean match = originModel.getJoinsGraph().match(expectModel.getJoinsGraph(), matchAlias);
diff --git a/src/query/pom.xml b/src/query/pom.xml
index 1479d73d1f..a970b1e41d 100644
--- a/src/query/pom.xml
+++ b/src/query/pom.xml
@@ -64,6 +64,10 @@
<groupId>org.apache.calcite</groupId>
<artifactId>calcite-linq4j</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.postgresql</groupId>
+ <artifactId>postgresql</artifactId>
+ </dependency>
<dependency>
<groupId>commons-collections</groupId>
diff --git a/src/server/pom.xml b/src/server/pom.xml
index 755e7c6ce4..d6a413ec2c 100644
--- a/src/server/pom.xml
+++ b/src/server/pom.xml
@@ -69,6 +69,10 @@
<groupId>org.apache.kylin</groupId>
<artifactId>kap-second-storage-ui</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.apache.kylin</groupId>
+ <artifactId>distributed-lock-ext</artifactId>
+ </dependency>
<!-- https://mvnrepository.com/artifact/org.scala-lang.modules/scala-xml -->
<dependency>
@@ -177,6 +181,26 @@
<groupId>com.taobao.arthas</groupId>
<artifactId>arthas-spring-boot-starter</artifactId>
</dependency>
+ <dependency>
+ <groupId>io.micrometer</groupId>
+ <artifactId>micrometer-registry-prometheus</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.postgresql</groupId>
+ <artifactId>postgresql</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.lmax</groupId>
+ <artifactId>disruptor</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.latencyutils</groupId>
+ <artifactId>LatencyUtils</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.javassist</groupId>
+ <artifactId>javassist</artifactId>
+ </dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
diff --git a/src/server/src/main/java/org/apache/kylin/rest/QueryNodeFilter.java b/src/server/src/main/java/org/apache/kylin/rest/QueryNodeFilter.java
index 54c07d3fbb..50e7276660 100644
--- a/src/server/src/main/java/org/apache/kylin/rest/QueryNodeFilter.java
+++ b/src/server/src/main/java/org/apache/kylin/rest/QueryNodeFilter.java
@@ -53,7 +53,7 @@ import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.persistence.transaction.UnitOfWork;
import org.apache.kylin.common.util.JsonUtil;
import org.apache.kylin.common.util.Pair;
-import org.apache.kylin.common.util.Unsafe;
+import org.apache.kylin.metadata.epoch.EpochManager;
import org.apache.kylin.metadata.project.NProjectManager;
import org.apache.kylin.metadata.resourcegroup.ResourceGroupManager;
import org.apache.kylin.rest.cluster.ClusterManager;
@@ -76,7 +76,6 @@ import org.springframework.web.context.request.ServletRequestAttributes;
import com.google.common.collect.Sets;
-import org.apache.kylin.metadata.epoch.EpochManager;
import lombok.val;
import lombok.extern.slf4j.Slf4j;
@@ -236,7 +235,7 @@ public class QueryNodeFilter implements Filter {
responseStatus = HttpServletResponse.SC_INTERNAL_SERVER_ERROR;
Message msg = MsgPicker.getMsg();
KylinException exception = getKylinException(project, msg);
- ErrorResponse errorResponse = new ErrorResponse(Unsafe.getUrlFromHttpServletRequest(servletRequest),
+ ErrorResponse errorResponse = new ErrorResponse(servletRequest.getRequestURL().toString(),
exception);
responseBody = JsonUtil.writeValueAsBytes(errorResponse);
responseHeaders = new HttpHeaders();
@@ -358,7 +357,7 @@ public class QueryNodeFilter implements Filter {
public void writeConnectionErrorResponse(HttpServletRequest servletRequest, HttpServletResponse servletResponse)
throws IOException {
- ErrorResponse errorResponse = new ErrorResponse(Unsafe.getUrlFromHttpServletRequest(servletRequest),
+ ErrorResponse errorResponse = new ErrorResponse(servletRequest.getRequestURL().toString(),
new KylinException(FAILED_CONNECT_CATALOG, MsgPicker.getMsg().getConnectDatabaseError(), false));
byte[] responseBody = JsonUtil.writeValueAsBytes(errorResponse);
HttpHeaders responseHeaders = new HttpHeaders();
diff --git a/src/spark-project/engine-build-sdk/pom.xml b/src/spark-project/engine-build-sdk/pom.xml
index 0e2b3d6866..da144f3e7d 100644
--- a/src/spark-project/engine-build-sdk/pom.xml
+++ b/src/spark-project/engine-build-sdk/pom.xml
@@ -29,7 +29,6 @@
<name>Kylin - Engine Build SDK</name>
<artifactId>kylin-engine-build-sdk</artifactId>
-
<dependencies>
<dependency>
<groupId>org.projectlombok</groupId>
diff --git a/src/spark-project/engine-spark/pom.xml b/src/spark-project/engine-spark/pom.xml
index b03cffa85b..2b4edf32e3 100644
--- a/src/spark-project/engine-spark/pom.xml
+++ b/src/spark-project/engine-spark/pom.xml
@@ -62,6 +62,10 @@
<groupId>org.apache.kylin</groupId>
<artifactId>kylin-engine-build-sdk</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.apache.kylin</groupId>
+ <artifactId>distributed-lock-ext</artifactId>
+ </dependency>
<dependency>
<groupId>io.kyligence.ke</groupId>
diff --git a/src/spark-project/engine-spark/src/test/java/org/apache/kylin/engine/spark/job/JobManagerTest.java b/src/spark-project/engine-spark/src/test/java/org/apache/kylin/engine/spark/job/JobManagerTest.java
index 145e017504..805e8ae20f 100644
--- a/src/spark-project/engine-spark/src/test/java/org/apache/kylin/engine/spark/job/JobManagerTest.java
+++ b/src/spark-project/engine-spark/src/test/java/org/apache/kylin/engine/spark/job/JobManagerTest.java
@@ -35,6 +35,9 @@ import java.util.Set;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.exception.KylinException;
+import org.apache.kylin.common.persistence.transaction.UnitOfWork;
+import org.apache.kylin.common.util.NLocalFileMetadataTestCase;
+import org.apache.kylin.engine.spark.ExecutableUtils;
import org.apache.kylin.job.engine.JobEngineConfig;
import org.apache.kylin.job.execution.AbstractExecutable;
import org.apache.kylin.job.execution.ExecutableParams;
@@ -42,11 +45,6 @@ import org.apache.kylin.job.execution.NExecutableManager;
import org.apache.kylin.job.impl.threadpool.NDefaultScheduler;
import org.apache.kylin.job.manager.JobManager;
import org.apache.kylin.job.model.JobParam;
-import org.apache.kylin.metadata.model.SegmentRange;
-import org.apache.kylin.metadata.model.SegmentStatusEnum;
-import org.apache.kylin.common.persistence.transaction.UnitOfWork;
-import org.apache.kylin.common.util.NLocalFileMetadataTestCase;
-import org.apache.kylin.engine.spark.ExecutableUtils;
import org.apache.kylin.metadata.cube.model.IndexEntity;
import org.apache.kylin.metadata.cube.model.IndexPlan;
import org.apache.kylin.metadata.cube.model.LayoutEntity;
@@ -60,6 +58,8 @@ import org.apache.kylin.metadata.cube.model.NIndexPlanManager;
import org.apache.kylin.metadata.cube.model.PartitionStatusEnum;
import org.apache.kylin.metadata.job.JobBucket;
import org.apache.kylin.metadata.model.NDataModelManager;
+import org.apache.kylin.metadata.model.SegmentRange;
+import org.apache.kylin.metadata.model.SegmentStatusEnum;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Rule;
@@ -353,17 +353,18 @@ public class JobManagerTest extends NLocalFileMetadataTestCase {
@Test
public void testQuotaLimitReached() {
- thrown.expect(KylinException.class);
NDefaultScheduler defaultScheduler = NDefaultScheduler.getInstance(PROJECT);
- defaultScheduler.init(new JobEngineConfig(KylinConfig.getInstanceFromEnv()));
+ defaultScheduler.init(new JobEngineConfig(getTestConfig()));
defaultScheduler.getContext().setReachQuotaLimit(true);
- JobParam param = new JobParam(Sets.newHashSet(), null, null, "ADMIn", Sets.newHashSet(), null);
- try {
- jobManager.addJob(param);
- } finally {
- defaultScheduler.forceShutdown();
- defaultScheduler.getContext().setReachQuotaLimit(false);
- }
+ Assert.assertThrows(KylinException.class, () -> {
+ try {
+ JobManager.checkStorageQuota(PROJECT);
+ } finally {
+ defaultScheduler.forceShutdown();
+ defaultScheduler.getContext().setReachQuotaLimit(false);
+ }
+ });
+
}
@Test
diff --git a/src/spark-project/spark-common/pom.xml b/src/spark-project/spark-common/pom.xml
index ff80077128..9556bd21d1 100644
--- a/src/spark-project/spark-common/pom.xml
+++ b/src/spark-project/spark-common/pom.xml
@@ -140,6 +140,11 @@
<artifactId>curator-test</artifactId>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.awaitility</groupId>
+ <artifactId>awaitility</artifactId>
+ <scope>test</scope>
+ </dependency>
<dependency>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin-soft-affinity-cache</artifactId>
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/asyncprofiler/AsyncArchUtil.java b/src/spark-project/spark-common/src/main/java/org/apache/kylin/common/asyncprofiler/AsyncArchUtil.java
similarity index 100%
rename from src/core-common/src/main/java/org/apache/kylin/common/asyncprofiler/AsyncArchUtil.java
rename to src/spark-project/spark-common/src/main/java/org/apache/kylin/common/asyncprofiler/AsyncArchUtil.java
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/asyncprofiler/AsyncProfiler.java b/src/spark-project/spark-common/src/main/java/org/apache/kylin/common/asyncprofiler/AsyncProfiler.java
similarity index 94%
rename from src/core-common/src/main/java/org/apache/kylin/common/asyncprofiler/AsyncProfiler.java
rename to src/spark-project/spark-common/src/main/java/org/apache/kylin/common/asyncprofiler/AsyncProfiler.java
index a0ba5f13a1..1c0b3a795e 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/asyncprofiler/AsyncProfiler.java
+++ b/src/spark-project/spark-common/src/main/java/org/apache/kylin/common/asyncprofiler/AsyncProfiler.java
@@ -47,18 +47,6 @@ public class AsyncProfiler {
return profiler;
}
- // for ut test
- public static AsyncProfiler utInstance() {
- if (profiler == null) {
- return new AsyncProfiler("");
- }
- return profiler;
- }
-
- private AsyncProfiler(String ignore) {
- logger.info("Test arg for ut: {}", ignore);
- }
-
private AsyncProfiler(boolean loadLocalLib) {
try {
boolean isTestingOnLocalMac = System.getProperty("os.name", "").contains("Mac")
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/asyncprofiler/AsyncProfilerUtils.java b/src/spark-project/spark-common/src/main/java/org/apache/kylin/common/asyncprofiler/AsyncProfilerUtils.java
similarity index 100%
rename from src/core-common/src/main/java/org/apache/kylin/common/asyncprofiler/AsyncProfilerUtils.java
rename to src/spark-project/spark-common/src/main/java/org/apache/kylin/common/asyncprofiler/AsyncProfilerUtils.java
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/source/dfs/DFSFileTable.java b/src/spark-project/spark-common/src/main/java/org/apache/kylin/source/dfs/DFSFileTable.java
similarity index 100%
rename from src/core-metadata/src/main/java/org/apache/kylin/source/dfs/DFSFileTable.java
rename to src/spark-project/spark-common/src/main/java/org/apache/kylin/source/dfs/DFSFileTable.java
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/source/dfs/DFSFileTableReader.java b/src/spark-project/spark-common/src/main/java/org/apache/kylin/source/dfs/DFSFileTableReader.java
similarity index 98%
rename from src/core-metadata/src/main/java/org/apache/kylin/source/dfs/DFSFileTableReader.java
rename to src/spark-project/spark-common/src/main/java/org/apache/kylin/source/dfs/DFSFileTableReader.java
index 0a2d2a2ffb..17f63712ae 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/source/dfs/DFSFileTableReader.java
+++ b/src/spark-project/spark-common/src/main/java/org/apache/kylin/source/dfs/DFSFileTableReader.java
@@ -204,7 +204,7 @@ public class DFSFileTableReader implements TableReader {
String nextLine() throws IOException; // return null on EOF
}
- private class SeqRowReader implements RowReader {
+ private static class SeqRowReader implements RowReader {
Reader reader;
Writable key;
Text value;
@@ -230,7 +230,7 @@ public class DFSFileTableReader implements TableReader {
}
}
- private class CsvRowReader implements RowReader {
+ private static class CsvRowReader implements RowReader {
BufferedReader reader;
CsvRowReader(FileSystem fs, String path) throws IOException {
diff --git a/src/spark-project/spark-common/src/main/resources/async-profiler-lib/libasyncProfiler-linux-arm64.so b/src/spark-project/spark-common/src/main/resources/async-profiler-lib/libasyncProfiler-linux-arm64.so
new file mode 100644
index 0000000000..b959823506
Binary files /dev/null and b/src/spark-project/spark-common/src/main/resources/async-profiler-lib/libasyncProfiler-linux-arm64.so differ
diff --git a/src/spark-project/spark-common/src/main/resources/async-profiler-lib/libasyncProfiler-linux-x64.so b/src/spark-project/spark-common/src/main/resources/async-profiler-lib/libasyncProfiler-linux-x64.so
new file mode 100644
index 0000000000..6d961cec0d
Binary files /dev/null and b/src/spark-project/spark-common/src/main/resources/async-profiler-lib/libasyncProfiler-linux-x64.so differ
diff --git a/src/spark-project/spark-common/src/main/resources/async-profiler-lib/libasyncProfiler-mac.so b/src/spark-project/spark-common/src/main/resources/async-profiler-lib/libasyncProfiler-mac.so
new file mode 100644
index 0000000000..ad45237d5f
Binary files /dev/null and b/src/spark-project/spark-common/src/main/resources/async-profiler-lib/libasyncProfiler-mac.so differ
diff --git a/src/spark-project/spark-common/src/main/resources/async-profiler-lib/linux64/libasyncProfiler.so b/src/spark-project/spark-common/src/main/resources/async-profiler-lib/linux64/libasyncProfiler.so
new file mode 100755
index 0000000000..4153f52868
Binary files /dev/null and b/src/spark-project/spark-common/src/main/resources/async-profiler-lib/linux64/libasyncProfiler.so differ
diff --git a/src/spark-project/spark-common/src/main/resources/async-profiler-lib/macOS/libasyncProfiler.so b/src/spark-project/spark-common/src/main/resources/async-profiler-lib/macOS/libasyncProfiler.so
new file mode 100755
index 0000000000..c7298c6b82
Binary files /dev/null and b/src/spark-project/spark-common/src/main/resources/async-profiler-lib/macOS/libasyncProfiler.so differ
diff --git a/src/core-common/src/main/scala/org/apache/kylin/common/asyncprofiler/AsyncProfilerExecutorPlugin.scala b/src/spark-project/spark-common/src/main/scala/org/apache/kylin/common/asyncprofiler/AsyncProfilerExecutorPlugin.scala
similarity index 100%
rename from src/core-common/src/main/scala/org/apache/kylin/common/asyncprofiler/AsyncProfilerExecutorPlugin.scala
rename to src/spark-project/spark-common/src/main/scala/org/apache/kylin/common/asyncprofiler/AsyncProfilerExecutorPlugin.scala
diff --git a/src/core-common/src/main/scala/org/apache/kylin/common/asyncprofiler/AsyncProfilerTool.scala b/src/spark-project/spark-common/src/main/scala/org/apache/kylin/common/asyncprofiler/AsyncProfilerTool.scala
similarity index 99%
rename from src/core-common/src/main/scala/org/apache/kylin/common/asyncprofiler/AsyncProfilerTool.scala
rename to src/spark-project/spark-common/src/main/scala/org/apache/kylin/common/asyncprofiler/AsyncProfilerTool.scala
index 69bb7c72bb..d618896946 100644
--- a/src/core-common/src/main/scala/org/apache/kylin/common/asyncprofiler/AsyncProfilerTool.scala
+++ b/src/spark-project/spark-common/src/main/scala/org/apache/kylin/common/asyncprofiler/AsyncProfilerTool.scala
@@ -20,7 +20,7 @@ package org.apache.kylin.common.asyncprofiler
import org.slf4j.{Logger, LoggerFactory}
-/**
+/*
* this class is not thread safe
*/
object AsyncProfilerTool {
diff --git a/src/core-common/src/main/scala/org/apache/kylin/common/asyncprofiler/Message.scala b/src/spark-project/spark-common/src/main/scala/org/apache/kylin/common/asyncprofiler/Message.scala
similarity index 98%
rename from src/core-common/src/main/scala/org/apache/kylin/common/asyncprofiler/Message.scala
rename to src/spark-project/spark-common/src/main/scala/org/apache/kylin/common/asyncprofiler/Message.scala
index cd3019d528..881992904b 100644
--- a/src/core-common/src/main/scala/org/apache/kylin/common/asyncprofiler/Message.scala
+++ b/src/spark-project/spark-common/src/main/scala/org/apache/kylin/common/asyncprofiler/Message.scala
@@ -18,7 +18,8 @@
package org.apache.kylin.common.asyncprofiler
-/** simple command in string
+/*
+ * simple command in string
* messages are in form of {command}{executor-id}:{param}
* commands are limited to fixed 3 characters
*/
@@ -68,4 +69,3 @@ object Message {
(getCommand(msg), getId(msg), getParam(msg))
}
}
-
diff --git a/src/core-common/src/test/java/org/apache/kylin/common/asyncprofiler/AsyncArchUtilTest.java b/src/spark-project/spark-common/src/test/java/org/apache/kylin/common/asyncprofiler/AsyncArchUtilTest.java
similarity index 100%
rename from src/core-common/src/test/java/org/apache/kylin/common/asyncprofiler/AsyncArchUtilTest.java
rename to src/spark-project/spark-common/src/test/java/org/apache/kylin/common/asyncprofiler/AsyncArchUtilTest.java
diff --git a/src/core-common/src/test/java/org/apache/kylin/common/asyncprofiler/AsyncProfilerTest.java b/src/spark-project/spark-common/src/test/java/org/apache/kylin/common/asyncprofiler/AsyncProfilerTest.java
similarity index 79%
rename from src/core-common/src/test/java/org/apache/kylin/common/asyncprofiler/AsyncProfilerTest.java
rename to src/spark-project/spark-common/src/test/java/org/apache/kylin/common/asyncprofiler/AsyncProfilerTest.java
index 176d9ad119..c819afc4f5 100644
--- a/src/core-common/src/test/java/org/apache/kylin/common/asyncprofiler/AsyncProfilerTest.java
+++ b/src/spark-project/spark-common/src/test/java/org/apache/kylin/common/asyncprofiler/AsyncProfilerTest.java
@@ -67,22 +67,4 @@ public class AsyncProfilerTest {
AsyncProfiler.getInstance(true)::stop);
}
- @Test
- public void testAsyncProfilerUtInstance() {
- AsyncProfiler originInstance = AsyncProfiler.getInstance(true);
- AsyncProfiler utInstance = AsyncProfiler.utInstance();
- Assert.assertSame(originInstance, utInstance);
- }
-
- @Test
- public void testStopUnload() {
- AsyncProfiler asyncProfiler = AsyncProfiler.utInstance();
- Assert.assertFalse(asyncProfiler.isLoaded());
- asyncProfiler.stop();
- }
-
- @Test
- public void testExecuteUnload() throws IOException {
- Assert.assertEquals("", AsyncProfiler.utInstance().execute(""));
- }
}
diff --git a/src/core-common/src/test/java/org/apache/kylin/common/asyncprofiler/AsyncProfilerToolTest.java b/src/spark-project/spark-common/src/test/java/org/apache/kylin/common/asyncprofiler/AsyncProfilerToolTest.java
similarity index 100%
rename from src/core-common/src/test/java/org/apache/kylin/common/asyncprofiler/AsyncProfilerToolTest.java
rename to src/spark-project/spark-common/src/test/java/org/apache/kylin/common/asyncprofiler/AsyncProfilerToolTest.java
diff --git a/src/core-common/src/test/java/org/apache/kylin/common/asyncprofiler/AsyncProfilerUtilsTest.java b/src/spark-project/spark-common/src/test/java/org/apache/kylin/common/asyncprofiler/AsyncProfilerUtilsTest.java
similarity index 96%
rename from src/core-common/src/test/java/org/apache/kylin/common/asyncprofiler/AsyncProfilerUtilsTest.java
rename to src/spark-project/spark-common/src/test/java/org/apache/kylin/common/asyncprofiler/AsyncProfilerUtilsTest.java
index fba1ad54a6..90c4423b69 100644
--- a/src/core-common/src/test/java/org/apache/kylin/common/asyncprofiler/AsyncProfilerUtilsTest.java
+++ b/src/spark-project/spark-common/src/test/java/org/apache/kylin/common/asyncprofiler/AsyncProfilerUtilsTest.java
@@ -18,7 +18,6 @@
package org.apache.kylin.common.asyncprofiler;
-import static org.awaitility.Awaitility.await;
import static org.mockito.Mockito.mock;
import java.io.File;
@@ -30,6 +29,7 @@ import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
+import org.awaitility.Awaitility;
import org.awaitility.Duration;
import org.junit.Assert;
import org.junit.Test;
@@ -82,7 +82,7 @@ public class AsyncProfilerUtilsTest {
ExecutorService cachedThreadPool = Executors.newCachedThreadPool();
Thread t1 = new Thread(() -> {
- await().pollDelay(new Duration(1, TimeUnit.SECONDS)).until(() -> true);
+ Awaitility.await().pollDelay(new Duration(1, TimeUnit.SECONDS)).until(() -> true);
for (int i = 0; i < 3; i++) {
cachedThreadPool.execute(() -> asyncProfilerUtils.cachedResult.countDown());
}
@@ -100,7 +100,7 @@ public class AsyncProfilerUtilsTest {
ExecutorService cachedThreadPool = Executors.newCachedThreadPool();
Thread t1 = new Thread(() -> {
- await().pollDelay(new Duration(1, TimeUnit.MILLISECONDS)).until(() -> true);
+ Awaitility.await().pollDelay(new Duration(1, TimeUnit.MILLISECONDS)).until(() -> true);
for (int i = 0; i < 3; i++) {
cachedThreadPool.execute(() -> asyncProfilerUtils.cachedResult.countDown());
}
diff --git a/src/core-common/src/test/java/org/apache/kylin/common/asyncprofiler/MessageTest.java b/src/spark-project/spark-common/src/test/java/org/apache/kylin/common/asyncprofiler/MessageTest.java
similarity index 96%
rename from src/core-common/src/test/java/org/apache/kylin/common/asyncprofiler/MessageTest.java
rename to src/spark-project/spark-common/src/test/java/org/apache/kylin/common/asyncprofiler/MessageTest.java
index eef0290854..dddaeaae68 100644
--- a/src/core-common/src/test/java/org/apache/kylin/common/asyncprofiler/MessageTest.java
+++ b/src/spark-project/spark-common/src/test/java/org/apache/kylin/common/asyncprofiler/MessageTest.java
@@ -37,7 +37,8 @@ public class MessageTest {
@Test
public void testExecutorMessage() {
- String executorMessage = Message.createExecutorMessage(Message.START(), "-0", AsyncProfilerToolTest.DUMP_PARAMS);
+ String executorMessage = Message.createExecutorMessage(Message.START(), "-0",
+ AsyncProfilerToolTest.DUMP_PARAMS);
Tuple3<String, String, String> tuple = Message.processMessage(executorMessage);
Assert.assertEquals("STA-0:flamegraph", executorMessage);
Assert.assertEquals(Message.START(), tuple._1());
diff --git a/src/systools/pom.xml b/src/systools/pom.xml
index c9026a1dac..f6e3cd1530 100644
--- a/src/systools/pom.xml
+++ b/src/systools/pom.xml
@@ -189,26 +189,11 @@
<artifactId>mapstruct</artifactId>
</dependency>
- <!-- Swagger -->
- <dependency>
- <groupId>io.springfox</groupId>
- <artifactId>springfox-boot-starter</artifactId>
- </dependency>
- <dependency>
- <groupId>io.springfox</groupId>
- <artifactId>springfox-boot-starter</artifactId>
- <version>3.0.0</version>
- </dependency>
-
- <dependency>
- <groupId>org.mapstruct</groupId>
- <artifactId>mapstruct</artifactId>
- </dependency>
-
<!-- util -->
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
+ <scope>provided</scope>
</dependency>
<dependency>
<groupId>redis.clients</groupId>
@@ -230,6 +215,11 @@
<artifactId>mockito-core</artifactId>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.javassist</groupId>
+ <artifactId>javassist</artifactId>
+ <scope>test</scope>
+ </dependency>
<dependency>
<groupId>org.powermock</groupId>
<artifactId>powermock-module-junit4</artifactId>
[kylin] 07/17: [DIRTY] fix favorite rules return null
Posted by xx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 8ada2027f468f7da49de6e7063ae1a3262d604e0
Author: Ruixuan Zhang <ru...@kyligence.io>
AuthorDate: Thu Nov 17 18:27:48 2022 +0800
[DIRTY] fix favorite rules return null
---
.../src/main/java/org/apache/kylin/rest/service/ProjectService.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/common-service/src/main/java/org/apache/kylin/rest/service/ProjectService.java b/src/common-service/src/main/java/org/apache/kylin/rest/service/ProjectService.java
index 10c5b19f6f..93d8cad3ba 100644
--- a/src/common-service/src/main/java/org/apache/kylin/rest/service/ProjectService.java
+++ b/src/common-service/src/main/java/org/apache/kylin/rest/service/ProjectService.java
@@ -586,7 +586,7 @@ public class ProjectService extends BasicService {
response.setPrincipal(projectInstance.getPrincipal());
// return favorite rules
- // TODO: adapt
+ response.setFavoriteRules(projectSmartService != null ? projectSmartService.getFavoriteRules(project) : null);
response.setScd2Enabled(config.isQueryNonEquiJoinModelEnabled());
[kylin] 09/17: KYLIN-5395 sync column comment to dimension name and measure comment (#29499)
Posted by xx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit ae22f91de94d4e399f4498e51876e6ca9cd28712
Author: fanfanAlice <41...@users.noreply.github.com>
AuthorDate: Sun Nov 20 11:44:47 2022 +0800
KYLIN-5395 sync column comment to dimension name and measure comment (#29499)
* sync column comment to dimension name and measure comment
* bug fix for special char replace
Co-authored-by: fanfanAlice <18...@163.com>
---
.../apache/kylin/metadata/model/ColumnDesc.java | 8 +
.../org/apache/kylin/metadata/model/TblColRef.java | 4 +-
.../rest/controller/open/OpenModelController.java | 17 ++
.../controller/open/OpenModelControllerTest.java | 16 ++
.../response/SynchronizedCommentsResponse.java | 220 +++++++++++++++++++++
.../apache/kylin/rest/service/ModelService.java | 8 +-
.../kylin/rest/service/ModelServiceTest.java | 95 +++++++++
7 files changed, 362 insertions(+), 6 deletions(-)
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
index a5968b6cf5..e179e2bb63 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
@@ -268,6 +268,14 @@ public class ColumnDesc implements Serializable {
return desc;
}
+ public String getCanonicalName() {
+ String tableName = null;
+ if (table != null) {
+ tableName = table.getIdentity();
+ }
+ return tableName + "." + getName();
+ }
+
@Override
public int hashCode() {
final int prime = 31;
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
index d4764ff2e2..13022d135d 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java
@@ -203,7 +203,7 @@ public class TblColRef implements Serializable {
this.table = table;
this.column = column;
}
-
+
private String wrapIdentity(String wrap) {
return wrap + getTableAlias() + wrap + "." + wrap + getName() + wrap;
}
@@ -281,7 +281,7 @@ public class TblColRef implements Serializable {
}
public String getCanonicalName() {
- return getTable() + "." + getName();
+ return column.getCanonicalName();
}
public String getDatatype() {
diff --git a/src/metadata-server/src/main/java/io/kyligence/kap/rest/controller/open/OpenModelController.java b/src/metadata-server/src/main/java/io/kyligence/kap/rest/controller/open/OpenModelController.java
index d9642ade0c..23c16ef30a 100644
--- a/src/metadata-server/src/main/java/io/kyligence/kap/rest/controller/open/OpenModelController.java
+++ b/src/metadata-server/src/main/java/io/kyligence/kap/rest/controller/open/OpenModelController.java
@@ -68,6 +68,7 @@ import org.apache.kylin.rest.response.IndexResponse;
import org.apache.kylin.rest.response.NModelDescResponse;
import org.apache.kylin.rest.response.OpenGetIndexResponse;
import org.apache.kylin.rest.response.OpenGetIndexResponse.IndexDetail;
+import org.apache.kylin.rest.response.SynchronizedCommentsResponse;
import org.apache.kylin.rest.service.FusionIndexService;
import org.apache.kylin.rest.service.FusionModelService;
import org.apache.kylin.rest.service.ModelService;
@@ -111,6 +112,8 @@ public class OpenModelController extends NBasicController {
.collect(Collectors.toSet());
public static final String MODEL_ID = "modelId";
+ public static final String FACT_TABLE = "fact_table";
+
@Autowired
private NModelController modelController;
@@ -493,4 +496,18 @@ public class OpenModelController extends NBasicController {
throw new KylinException(FAILED_UPDATE_MODEL, root);
}
}
+
+ @ApiOperation(value = "comments synchronization", tags = { "AI" })
+ @PostMapping(value = "/comments_synchronization")
+ @ResponseBody
+ public EnvelopeResponse<SynchronizedCommentsResponse> commentsSynchronization(
+ @RequestBody ModelRequest modelRequest) {
+ modelRequest.setProject(checkProjectName(modelRequest.getProject()));
+ checkRequiredArg(ALIAS, modelRequest.getRawAlias());
+ checkRequiredArg(FACT_TABLE, modelRequest.getRootFactTableName());
+ SynchronizedCommentsResponse synchronizedCommentsResponse = new SynchronizedCommentsResponse();
+ synchronizedCommentsResponse.syncComment(modelRequest);
+ modelService.checkBeforeModelSave(synchronizedCommentsResponse.getModelRequest());
+ return new EnvelopeResponse<>(KylinException.CODE_SUCCESS, synchronizedCommentsResponse, "");
+ }
}
diff --git a/src/metadata-server/src/test/java/io/kyligence/kap/rest/controller/open/OpenModelControllerTest.java b/src/metadata-server/src/test/java/io/kyligence/kap/rest/controller/open/OpenModelControllerTest.java
index e36ccf48de..ccc7864448 100644
--- a/src/metadata-server/src/test/java/io/kyligence/kap/rest/controller/open/OpenModelControllerTest.java
+++ b/src/metadata-server/src/test/java/io/kyligence/kap/rest/controller/open/OpenModelControllerTest.java
@@ -641,4 +641,20 @@ public class OpenModelControllerTest extends NLocalFileMetadataTestCase {
.andExpect(MockMvcResultMatchers.status().isOk());
Mockito.verify(openModelController).updateSemantic(Mockito.any(OpenModelRequest.class));
}
+
+ @Test
+ public void testCommentsSynchronization() throws Exception {
+ String project = "default";
+ String modelName = "model1";
+ ModelRequest modelRequest = new ModelRequest();
+ modelRequest.setProject(project);
+ modelRequest.setAlias(modelName);
+ modelRequest.setRootFactTableName("test_fact_table");
+
+ mockMvc.perform(MockMvcRequestBuilders.post("/api/models/comments_synchronization")
+ .contentType(MediaType.APPLICATION_JSON).content(JsonUtil.writeValueAsString(modelRequest))
+ .accept(MediaType.parseMediaType(HTTP_VND_APACHE_KYLIN_V4_PUBLIC_JSON)))
+ .andExpect(MockMvcResultMatchers.status().isOk());
+ Mockito.verify(openModelController).commentsSynchronization(modelRequest);
+ }
}
diff --git a/src/modeling-service/src/main/java/org/apache/kylin/rest/response/SynchronizedCommentsResponse.java b/src/modeling-service/src/main/java/org/apache/kylin/rest/response/SynchronizedCommentsResponse.java
new file mode 100644
index 0000000000..3ec10cc795
--- /dev/null
+++ b/src/modeling-service/src/main/java/org/apache/kylin/rest/response/SynchronizedCommentsResponse.java
@@ -0,0 +1,220 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kylin.rest.response;
+
+import java.io.Serializable;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.lang.StringUtils;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.metadata.model.ColumnDesc;
+import org.apache.kylin.metadata.model.FunctionDesc;
+import org.apache.kylin.metadata.model.JoinTableDesc;
+import org.apache.kylin.metadata.model.NDataModel;
+import org.apache.kylin.metadata.model.NTableMetadataManager;
+import org.apache.kylin.metadata.model.TableDesc;
+import org.apache.kylin.rest.request.ModelRequest;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+
+@Data
+public class SynchronizedCommentsResponse {
+
+ @JsonProperty("model_request")
+ private ModelRequest modelRequest;
+
+ @JsonProperty("conflict_info")
+ private ConflictInfo conflictInfo;
+
+ @Data
+ @AllArgsConstructor
+ public static class ConflictInfo implements Serializable {
+ @JsonProperty("cols_with_same_comment")
+ private List<String> colsWithSameComment;
+ @JsonProperty("dims_origin_from_same_col")
+ private List<String> dimsOriginFromSameCol;
+ }
+
+ public static final Pattern SPECIAL_CHAR_PTN = Pattern.compile("((?![\\u4E00-\\u9FA5a-zA-Z0-9 _\\-()%?（）]+).)*");
+
+ public void syncComment(ModelRequest modelRequest) {
+ this.modelRequest = modelRequest;
+ Map<String, ColumnDesc> columnDescMap = getColumnDescMap();
+ syncDimensionNames(columnDescMap, modelRequest.getSimplifiedDimensions());
+ syncMeasureComments(columnDescMap, modelRequest.getSimplifiedMeasures());
+ }
+
+ private void syncDimensionNames(Map<String, ColumnDesc> columnDescMap,
+ List<NDataModel.NamedColumn> simplifiedDimensions) {
+ List<String> colsWithSameComment = Lists.newArrayList();
+ List<String> dimsOriginFromSameCol = Lists.newArrayList();
+ simplifiedDimensions.forEach(namedColumn -> {
+ String aliasDotColumn = StringUtils.upperCase(namedColumn.getAliasDotColumn());
+ ColumnDesc columnDesc = columnDescMap.get(aliasDotColumn);
+ String name = StringUtils.upperCase(namedColumn.getColTableName());
+ if (columnDesc == null) {
+ return;
+ }
+ String canonicalName = StringUtils.upperCase(columnDesc.getCanonicalName());
+ List<ColumnDesc> columnDescList = simplifiedDimensions.stream()
+ .map(simpleDimension -> columnDescMap
+ .get(StringUtils.upperCase(simpleDimension.getAliasDotColumn())))
+ .filter(Objects::nonNull)
+ .filter(column -> StringUtils.upperCase(column.getCanonicalName()).equals(canonicalName))
+ .collect(Collectors.toList());
+ long dimsOriginFromSameColCount = columnDescList.size();
+ long colsWithSameCommentCount = simplifiedDimensions.stream()
+ .map(simpleDimension -> columnDescMap
+ .get(StringUtils.upperCase(simpleDimension.getAliasDotColumn())))
+ .filter(Objects::nonNull)
+ .filter(column -> StringUtils.isNotBlank(column.getComment())
+ && StringUtils.isNotBlank(columnDesc.getComment())
+ && StringUtils.upperCase(column.getComment())
+ .equals(StringUtils.upperCase(columnDesc.getComment())))
+ .count();
+ if (dimsOriginFromSameColCount > 1) {
+ dimsOriginFromSameCol.add(name);
+ }
+ if (colsWithSameCommentCount > 1) {
+ colsWithSameComment.add(name);
+ }
+ if (dimsOriginFromSameColCount == 1 && colsWithSameCommentCount == 1
+ && StringUtils.isNotBlank(columnDesc.getComment())) {
+ name = columnDesc.getComment();
+ }
+ namedColumn.setName(name);
+ });
+ this.modelRequest.setSimplifiedDimensions(simplifiedDimensions);
+ this.conflictInfo = new ConflictInfo(colsWithSameComment, dimsOriginFromSameCol);
+ }
+
+ private void syncMeasureComments(Map<String, ColumnDesc> columnDescMap,
+ List<SimplifiedMeasure> simplifiedMeasures) {
+ simplifiedMeasures.forEach(simplifiedMeasure -> {
+ List<ParameterResponse> parameterResponses = simplifiedMeasure.getParameterValue().stream().filter(
+ parameterResponse -> !FunctionDesc.PARAMETER_TYPE_CONSTANT.equals(parameterResponse.getType()))
+ .collect(Collectors.toList());
+ if (CollectionUtils.isEmpty(parameterResponses)) {
+ return;
+ }
+ String aliasDotColumn = parameterResponses.get(0).getValue();
+ ColumnDesc columnDesc = columnDescMap.get(aliasDotColumn);
+ if (columnDesc != null && StringUtils.isNotBlank(columnDesc.getComment())) {
+ simplifiedMeasure.setComment(columnDesc.getComment());
+ }
+ });
+ modelRequest.setSimplifiedMeasures(simplifiedMeasures);
+ }
+
+ public Map<String, ColumnDesc> getColumnDescMap() {
+ Map<String, ColumnDesc> columnDescMap = Maps.newHashMap();
+ String project = modelRequest.getProject();
+ String factTableName = modelRequest.getRootFactTableName();
+ Set<String> computedColumnNames = modelRequest.getComputedColumnNames().stream().map(StringUtils::upperCase)
+ .collect(Collectors.toSet());
+ List<JoinTableDesc> joinTables = modelRequest.getJoinTables();
+ List<NDataModel.NamedColumn> simplifiedDimensions = modelRequest.getSimplifiedDimensions();
+ simplifiedDimensions.forEach(namedColumn -> putColumnDesc(project, computedColumnNames, factTableName,
+ joinTables, columnDescMap, namedColumn.getAliasDotColumn()));
+ List<SimplifiedMeasure> simplifiedMeasures = modelRequest.getSimplifiedMeasures();
+ simplifiedMeasures.forEach(simplifiedMeasure -> {
+ List<ParameterResponse> parameterResponses = simplifiedMeasure.getParameterValue().stream().filter(
+ parameterResponse -> !FunctionDesc.PARAMETER_TYPE_CONSTANT.equals(parameterResponse.getType()))
+ .collect(Collectors.toList());
+ if (CollectionUtils.isEmpty(parameterResponses)) {
+ return;
+ }
+ parameterResponses.forEach(parameterResponse -> putColumnDesc(project, computedColumnNames, factTableName,
+ joinTables, columnDescMap, parameterResponse.getValue()));
+ });
+ return columnDescMap;
+ }
+
+ private void putColumnDesc(String project, Set<String> computedColumnNames, String factTableName,
+ List<JoinTableDesc> joinTables, Map<String, ColumnDesc> columnDescMap, String aliasDotColumn) {
+ if (aliasDotColumn == null) {
+ return;
+ }
+ ColumnDesc columnDesc = getColumnDescByAliasDotColumn(project, StringUtils.upperCase(aliasDotColumn),
+ computedColumnNames, StringUtils.upperCase(factTableName), joinTables);
+ if (columnDesc != null) {
+ columnDescMap.put(aliasDotColumn, columnDesc);
+ }
+ }
+
+ private ColumnDesc getColumnDescByAliasDotColumn(String project, String aliasDotColumn,
+ Set<String> computedColumnNames, String factTableName, List<JoinTableDesc> joinTables) {
+ String[] aliasDotColumnSplit = aliasDotColumn.split("\\.");
+ if (aliasDotColumnSplit.length != 2) {
+ return null;
+ }
+ String tableName = aliasDotColumnSplit[0];
+ String columnName = aliasDotColumnSplit[1];
+ if (computedColumnNames.contains(aliasDotColumn) || computedColumnNames.contains(columnName)) {
+ return null;
+ }
+ String factTableAlias = factTableName;
+ if (factTableName.contains(".") && factTableName.split("\\.").length == 2) {
+ factTableAlias = factTableName.split("\\.")[1];
+ }
+ NTableMetadataManager mgr = NTableMetadataManager.getInstance(KylinConfig.getInstanceFromEnv(), project);
+ TableDesc tableDesc;
+ if (tableName.equals(factTableAlias)) {
+ tableDesc = mgr.getTableDesc(factTableName);
+ } else {
+ Optional<JoinTableDesc> joinTable = joinTables.stream()
+ .filter(joinTableDesc -> StringUtils.upperCase(joinTableDesc.getAlias()).equals(tableName))
+ .findFirst();
+ if (joinTable.isPresent()) {
+ tableDesc = mgr.getTableDesc(joinTable.get().getTable());
+ } else {
+ return null;
+ }
+ }
+ Optional<ColumnDesc> column = Arrays.stream(tableDesc.getColumns())
+ .filter(columnDesc -> StringUtils.upperCase(columnDesc.getName()).equals(columnName)).findFirst();
+ if (column.isPresent()) {
+ ColumnDesc columnDesc = column.get();
+ String comment = columnDesc.getComment();
+ if (comment != null) {
+ columnDesc.setComment(replaceInvalidCharacters(comment.trim()));
+ }
+ return columnDesc;
+ }
+ return null;
+ }
+
+ private String replaceInvalidCharacters(String name) {
+ Matcher matcher = SPECIAL_CHAR_PTN.matcher(name);
+ return matcher.replaceAll("");
+ }
+}
diff --git a/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelService.java b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelService.java
index ab859e89b0..77fd3c6c5f 100644
--- a/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelService.java
+++ b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelService.java
@@ -298,9 +298,9 @@ public class ModelService extends AbstractModelService implements TableModelSupp
private static final String LAST_MODIFY = "last_modify";
public static final String REC_COUNT = "recommendations_count";
- public static final String VALID_NAME_FOR_DIMENSION = "^[\\u4E00-\\u9FA5a-zA-Z0-9 _\\-()%?（）]+$";
+ public static final Pattern VALID_NAME_FOR_DIMENSION = Pattern.compile("^[\\u4E00-\\u9FA5a-zA-Z0-9 _\\-()%?（）]+$");
- public static final String VALID_NAME_FOR_MEASURE = "^[\\u4E00-\\u9FA5a-zA-Z0-9 _\\-()%?（）.]+$";
+ public static final Pattern VALID_NAME_FOR_MEASURE = Pattern.compile("^[\\u4E00-\\u9FA5a-zA-Z0-9 _\\-()%?（）.]+$");
private static final List<String> MODEL_CONFIG_BLOCK_LIST = Lists.newArrayList("kylin.index.rule-scheduler-data");
@@ -2149,7 +2149,7 @@ public class ModelService extends AbstractModelService implements TableModelSupp
dimension.setName(StringUtils.trim(dimension.getName()));
// check if the dimension name is valid
if (StringUtils.length(dimension.getName()) > maxModelDimensionMeasureNameLength
- || !Pattern.compile(VALID_NAME_FOR_DIMENSION).matcher(dimension.getName()).matches())
+ || !VALID_NAME_FOR_DIMENSION.matcher(dimension.getName()).matches())
throw new KylinException(INVALID_NAME,
String.format(Locale.ROOT, MsgPicker.getMsg().getInvalidDimensionName(), dimension.getName(),
maxModelDimensionMeasureNameLength));
@@ -2216,7 +2216,7 @@ public class ModelService extends AbstractModelService implements TableModelSupp
if (!KylinConfig.getInstanceFromEnv().isMeasureNameCheckEnabled()) {
return true;
}
- return Pattern.compile(VALID_NAME_FOR_MEASURE).matcher(measureName).matches();
+ return VALID_NAME_FOR_MEASURE.matcher(measureName).matches();
}
private boolean isDupMeasure(SimplifiedMeasure measure, SimplifiedMeasure measure1) {
diff --git a/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelServiceTest.java b/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelServiceTest.java
index e55b82cdfe..e8edf45716 100644
--- a/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelServiceTest.java
+++ b/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelServiceTest.java
@@ -71,6 +71,7 @@ import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Objects;
+import java.util.Optional;
import java.util.Set;
import java.util.TimeZone;
import java.util.TreeMap;
@@ -148,6 +149,7 @@ import org.apache.kylin.metadata.model.SegmentStatusEnum;
import org.apache.kylin.metadata.model.SegmentStatusEnumToDisplay;
import org.apache.kylin.metadata.model.Segments;
import org.apache.kylin.metadata.model.TableDesc;
+import org.apache.kylin.metadata.model.TableRef;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.model.VolatileRange;
import org.apache.kylin.metadata.model.util.ExpandableMeasureUtil;
@@ -185,6 +187,7 @@ import org.apache.kylin.rest.response.SegmentPartitionResponse;
import org.apache.kylin.rest.response.SimplifiedColumnResponse;
import org.apache.kylin.rest.response.SimplifiedMeasure;
import org.apache.kylin.rest.response.SimplifiedTableResponse;
+import org.apache.kylin.rest.response.SynchronizedCommentsResponse;
import org.apache.kylin.rest.util.AclEvaluate;
import org.apache.kylin.rest.util.AclPermissionUtil;
import org.apache.kylin.rest.util.AclUtil;
@@ -5588,4 +5591,96 @@ public class ModelServiceTest extends SourceTestCase {
measure.setParameterValue(Lists.newArrayList(parameterResponse));
return measure;
}
+
+ @Test
+ public void testCreateModelSyncDimensionOrMeasure() {
+ NDataModelManager modelManager = NDataModelManager.getInstance(KylinConfig.getInstanceFromEnv(), "default");
+ ModelRequest modelRequest = createModelRequest(modelManager);
+ SynchronizedCommentsResponse response = new SynchronizedCommentsResponse();
+ response.syncComment(modelRequest);
+ ModelRequest newModelRequest = response.getModelRequest();
+ long measureCount = newModelRequest.getSimplifiedMeasures().stream()
+ .filter(simplifiedMeasure -> simplifiedMeasure.getComment() != null
+ && simplifiedMeasure.getComment().contains("____"))
+ .count();
+ long dimensionCount = newModelRequest.getSimplifiedDimensions().stream()
+ .filter(namedColumn -> namedColumn.getName().contains("____")).count();
+ Assert.assertEquals(11, measureCount);
+ Assert.assertEquals(10, dimensionCount);
+ SynchronizedCommentsResponse.ConflictInfo conflictInfo = response.getConflictInfo();
+ Assert.assertEquals(2, conflictInfo.getColsWithSameComment().size());
+ Assert.assertEquals(20, conflictInfo.getDimsOriginFromSameCol().size());
+ }
+
+ private ModelRequest createModelRequest(NDataModelManager modelManager) {
+ NDataModel model = modelManager.getDataModelDesc("82fa7671-a935-45f5-8779-85703601f49a");
+ ModelRequest modelRequest = new ModelRequest(model);
+ modelRequest.setProject("default");
+ modelRequest.setAlias("test_model");
+ modelRequest.setRootFactTableName(model.getRootFactTableName());
+ modelRequest.setLastModified(0L);
+ modelRequest.setStart("0");
+ modelRequest.setEnd("100");
+ modelRequest.setUuid(null);
+
+ List<NamedColumn> oriAllNamedColumns = model.getAllNamedColumns();
+
+ List<SimplifiedMeasure> simplified_measures = model.getAllMeasures().stream().map(oldSimplifiedMeasure -> {
+ SimplifiedMeasure simplifiedMeasure = new SimplifiedMeasure();
+ simplifiedMeasure.setName(oldSimplifiedMeasure.getName());
+ simplifiedMeasure.setExpression(oldSimplifiedMeasure.getFunction().getExpression());
+ simplifiedMeasure.setReturnType(oldSimplifiedMeasure.getFunction().getReturnType());
+ List<ParameterResponse> parameterResponses = oldSimplifiedMeasure.getFunction().getParameters().stream()
+ .map(parameterDesc -> {
+ String value = parameterDesc.getValue();
+ String type = parameterDesc.getType();
+ ParameterResponse response = new ParameterResponse();
+ response.setType(type);
+ response.setValue(value);
+ return response;
+ }).collect(Collectors.toList());
+ simplifiedMeasure.setParameterValue(parameterResponses);
+ return simplifiedMeasure;
+ }).collect(Collectors.toList());
+
+ modelRequest.setSimplifiedMeasures(simplified_measures);
+ modelRequest.setSimplifiedDimensions(oriAllNamedColumns);
+
+ NTableMetadataManager tableMetadataManager = NTableMetadataManager.getInstance(KylinConfig.getInstanceFromEnv(),
+ "default");
+ TableDesc tableDesc = tableMetadataManager.getTableDesc("DEFAULT.TEST_KYLIN_FACT");
+ int length = tableDesc.getColumns().length;
+ ColumnDesc[] columns = new ColumnDesc[length];
+ for (int i = 0; i < length; i++) {
+ ColumnDesc column = tableDesc.getColumns()[i];
+ column.setComment(
+ column.getComment() == null ? column.getName() + "____" + i : column.getComment() + "____" + i);
+ columns[i] = column;
+ }
+ String comment = columns[2].getComment();
+ columns[3].setComment(comment);
+ tableDesc.setColumns(columns);
+ tableMetadataManager.updateTableDesc(tableDesc);
+ return modelRequest;
+ }
+
+ @Test
+ public void testGetCanonicalName() {
+ TblColRef colRef = TblColRef.newDynamicColumn("test");
+ Assert.assertEquals("NULL.TEST", colRef.getCanonicalName());
+ TblColRef innerColumn = TblColRef.newInnerColumn("test", TblColRef.InnerDataTypeEnum.AGGREGATION_TYPE);
+ Assert.assertEquals("DEFAULT._KYLIN_TABLE.TEST", innerColumn.getCanonicalName());
+ NDataModelManager modelManager = NDataModelManager.getInstance(KylinConfig.getInstanceFromEnv(), "default");
+ NDataModel model = modelManager.getDataModelDesc("82fa7671-a935-45f5-8779-85703601f49a");
+ List<JoinTableDesc> joinTables = model.getJoinTables();
+ if (joinTables.size() == 0) {
+ return;
+ }
+ TableRef tableRef = joinTables.get(0).getTableRef();
+ Optional<TblColRef> first = tableRef.getColumns().stream().findFirst();
+ if (first.isPresent()) {
+ TblColRef colRef1 = first.get();
+ Assert.assertEquals("DEFAULT.TEST_ORDER.ORDER_ID", colRef1.getCanonicalName());
+ }
+ }
}
[kylin] 08/17: KYLIN-5394 Project-level push-down query does not take effect (jdbc push-down query)
Posted by xx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 4bb90cbab34f4168aa6be1050388d10413dd0816
Author: Hang Jia <75...@qq.com>
AuthorDate: Sat Nov 19 11:08:56 2022 +0800
KYLIN-5394 Project-level push-down query does not take effect (jdbc push-down query)
* KYLIN-5394 Project-level push-down query does not take effect (jdbc push-down query)
* fix UT in gcp: add await time in checkSnapshotSourceTableStatsJsonFile
Co-authored-by: jlf <lo...@kyligence.io>
---
.../kylin/source/adhocquery/IPushDownRunner.java | 7 +++
.../sdk/datasource/PushDownRunnerSDKImpl.java | 7 ++-
.../sdk/datasource/PushdownRunnerSDKImplTest.java | 6 +--
.../org/apache/kylin/query/util/PushDownUtil.java | 6 +--
.../SnapshotSourceTableStatsServiceTest.scala | 7 ++-
.../kylin/query/pushdown/DataSourceConfig.java | 47 +++++++++++++++++
.../pushdown/JdbcPushDownConnectionManager.java | 26 ++++++---
.../query/pushdown/PushDownRunnerJdbcImpl.java | 11 ++--
.../query/pushdown/PushDownRunnerSparkImpl.java | 7 ++-
.../query/pushdown/PushDownRunnerJdbcImplTest.java | 61 ++++++++++++++++++----
.../pushdown/PushDownRunnerSparkImplTest.java | 12 ++---
11 files changed, 162 insertions(+), 35 deletions(-)
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/IPushDownRunner.java b/src/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/IPushDownRunner.java
index 558cca08d1..d442600a19 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/IPushDownRunner.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/source/adhocquery/IPushDownRunner.java
@@ -27,6 +27,13 @@ import com.google.common.collect.Lists;
public interface IPushDownRunner {
+ void init(KylinConfig config, String project);
+
+ /**
+ * @deprecated for compatibility with old plugins
+ * @param config
+ */
+ @Deprecated
void init(KylinConfig config);
/**
diff --git a/src/datasource-sdk/src/main/java/org/apache/kylin/sdk/datasource/PushDownRunnerSDKImpl.java b/src/datasource-sdk/src/main/java/org/apache/kylin/sdk/datasource/PushDownRunnerSDKImpl.java
index 9426ccbe5e..5660e92ec9 100644
--- a/src/datasource-sdk/src/main/java/org/apache/kylin/sdk/datasource/PushDownRunnerSDKImpl.java
+++ b/src/datasource-sdk/src/main/java/org/apache/kylin/sdk/datasource/PushDownRunnerSDKImpl.java
@@ -29,11 +29,11 @@ import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.QueryContext;
import org.apache.kylin.common.util.DBUtils;
+import org.apache.kylin.metadata.project.NProjectManager;
import org.apache.kylin.metadata.querymeta.SelectedColumnMeta;
import org.apache.kylin.sdk.datasource.framework.JdbcConnector;
import org.apache.kylin.sdk.datasource.framework.SourceConnectorFactory;
import org.apache.kylin.source.adhocquery.IPushDownRunner;
-import org.apache.kylin.metadata.project.NProjectManager;
import lombok.extern.slf4j.Slf4j;
@@ -46,6 +46,11 @@ public class PushDownRunnerSDKImpl implements IPushDownRunner {
dataSource = SourceConnectorFactory.getJdbcConnector(config);
}
+ @Override
+ public void init(KylinConfig config, String project) {
+ init(config);
+ }
+
@Override
public void executeQuery(String query, List<List<String>> returnRows, List<SelectedColumnMeta> returnColumnMeta,
String project) throws Exception {
diff --git a/src/datasource-sdk/src/test/java/org/apache/kylin/sdk/datasource/PushdownRunnerSDKImplTest.java b/src/datasource-sdk/src/test/java/org/apache/kylin/sdk/datasource/PushdownRunnerSDKImplTest.java
index a275b07491..df9a5a2599 100644
--- a/src/datasource-sdk/src/test/java/org/apache/kylin/sdk/datasource/PushdownRunnerSDKImplTest.java
+++ b/src/datasource-sdk/src/test/java/org/apache/kylin/sdk/datasource/PushdownRunnerSDKImplTest.java
@@ -20,10 +20,10 @@ package org.apache.kylin.sdk.datasource;
import java.util.List;
import org.apache.kylin.common.QueryContext;
+import org.apache.kylin.metadata.project.NProjectManager;
import org.apache.kylin.metadata.project.ProjectInstance;
import org.apache.kylin.metadata.querymeta.SelectedColumnMeta;
import org.apache.kylin.sdk.datasource.framework.JdbcConnectorTest;
-import org.apache.kylin.metadata.project.NProjectManager;
import org.junit.Assert;
import org.junit.Test;
@@ -38,7 +38,7 @@ public class PushdownRunnerSDKImplTest extends JdbcConnectorTest {
npr.updateProject(projectInstance);
PushDownRunnerSDKImpl pushDownRunnerSDK = new PushDownRunnerSDKImpl();
- pushDownRunnerSDK.init(getTestConfig());
+ pushDownRunnerSDK.init(getTestConfig(), projectInstance.getName());
List<List<String>> returnRows = Lists.newArrayList();
List<SelectedColumnMeta> returnColumnMeta = Lists.newArrayList();
String sql = "select count(*) from LINEORDER";
@@ -49,7 +49,7 @@ public class PushdownRunnerSDKImplTest extends JdbcConnectorTest {
@Test
public void testExecuteUpdate() throws Exception {
PushDownRunnerSDKImpl pushDownRunnerSDK = new PushDownRunnerSDKImpl();
- pushDownRunnerSDK.init(getTestConfig());
+ pushDownRunnerSDK.init(getTestConfig(), null);
String sql = "update SSB.LINEORDER set LO_TAX=1 where LO_ORDERKEY = 1";
pushDownRunnerSDK.executeUpdate(sql, null);
}
diff --git a/src/query-common/src/main/java/org/apache/kylin/query/util/PushDownUtil.java b/src/query-common/src/main/java/org/apache/kylin/query/util/PushDownUtil.java
index 9fad04509b..4707d9406a 100644
--- a/src/query-common/src/main/java/org/apache/kylin/query/util/PushDownUtil.java
+++ b/src/query-common/src/main/java/org/apache/kylin/query/util/PushDownUtil.java
@@ -107,7 +107,7 @@ public class PushDownUtil {
}
IPushDownRunner runner = (IPushDownRunner) ClassUtil.newInstance(kylinConfig.getPushDownRunnerClassName());
- runner.init(kylinConfig);
+ runner.init(kylinConfig, project);
logger.debug("Query Pushdown runner {}", runner);
// set pushdown engine in query context
@@ -213,7 +213,7 @@ public class PushDownUtil {
// pushdown
IPushDownRunner runner = (IPushDownRunner) ClassUtil
.newInstance(kylinConfig.getPartitionCheckRunnerClassNameWithDefaultValue());
- runner.init(kylinConfig);
+ runner.init(kylinConfig, project);
runner.executeQuery(sql, returnRows, returnColumnMeta, project);
return Pair.newPair(returnRows, returnColumnMeta);
@@ -222,7 +222,7 @@ public class PushDownUtil {
public static void trySimplePushDownExecute(String sql, String project) throws Exception {
KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
IPushDownRunner runner = (IPushDownRunner) ClassUtil.newInstance(kylinConfig.getPushDownRunnerClassName());
- runner.init(kylinConfig);
+ runner.init(kylinConfig, project);
runner.executeUpdate(sql, project);
}
diff --git a/src/query-server/src/test/scala/org/apache/kylin/rest/service/SnapshotSourceTableStatsServiceTest.scala b/src/query-server/src/test/scala/org/apache/kylin/rest/service/SnapshotSourceTableStatsServiceTest.scala
index 6663a71041..5e0fdb3dc3 100644
--- a/src/query-server/src/test/scala/org/apache/kylin/rest/service/SnapshotSourceTableStatsServiceTest.scala
+++ b/src/query-server/src/test/scala/org/apache/kylin/rest/service/SnapshotSourceTableStatsServiceTest.scala
@@ -179,7 +179,7 @@ class SnapshotSourceTableStatsServiceTest extends SparderBaseFunSuite with Local
+ Constants.SOURCE_TABLE_STATS + "/" + tableIdentity)
try {
val out = fs.create(statsFile, true)
- try out.write(new String().getBytes())
+ try out.write(new String("{}").getBytes())
catch {
case e: IOException =>
log.error(s"overwrite stats file [$statsFile] failed!", e)
@@ -482,6 +482,8 @@ class SnapshotSourceTableStatsServiceTest extends SparderBaseFunSuite with Local
val tableIdentity = table.qualifiedName.toLowerCase(Locale.ROOT)
writeEmptyJsonFile(tableIdentity)
writeMarkFile()
+ await.pollDelay(Duration.ONE_SECOND).until(() => true)
+
var checkStatsFile = snapshotSourceTableStatsService.checkSnapshotSourceTableStatsJsonFile(DEFAULT_PROJECT, tableIdentity)
assertFalse(checkStatsFile)
@@ -507,6 +509,8 @@ class SnapshotSourceTableStatsServiceTest extends SparderBaseFunSuite with Local
val tableIdentity = table.qualifiedName.toLowerCase(Locale.ROOT)
writeEmptyJsonFile(tableIdentity)
writeMarkFile()
+ await.pollDelay(Duration.ONE_SECOND).until(() => true)
+
var checkStatsFile = snapshotSourceTableStatsService.checkSnapshotSourceTableStatsJsonFile(DEFAULT_PROJECT, tableIdentity)
assertFalse(checkStatsFile)
@@ -516,6 +520,7 @@ class SnapshotSourceTableStatsServiceTest extends SparderBaseFunSuite with Local
checkStatsFile = snapshotSourceTableStatsService.checkSnapshotSourceTableStatsJsonFile(DEFAULT_PROJECT, tableIdentity)
assertTrue(checkStatsFile)
+ await.pollDelay(Duration.ONE_SECOND).until(() => true)
writeMarkFile()
checkStatsFile = snapshotSourceTableStatsService.checkSnapshotSourceTableStatsJsonFile(DEFAULT_PROJECT, tableIdentity)
assertFalse(checkStatsFile)
diff --git a/src/spark-project/sparder/src/main/java/org/apache/kylin/query/pushdown/DataSourceConfig.java b/src/spark-project/sparder/src/main/java/org/apache/kylin/query/pushdown/DataSourceConfig.java
new file mode 100644
index 0000000000..dd0955f616
--- /dev/null
+++ b/src/spark-project/sparder/src/main/java/org/apache/kylin/query/pushdown/DataSourceConfig.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.query.pushdown;
+
+import org.apache.kylin.common.KylinConfig;
+
+import lombok.EqualsAndHashCode;
+
+/**
+ * @author hang.jia
+ */
+@EqualsAndHashCode
+public class DataSourceConfig {
+ String driverClassName;
+ String url;
+ String username;
+ String password;
+ int maxActive;
+ int maxIdle;
+ int minIdle;
+
+ public DataSourceConfig(KylinConfig config) {
+ driverClassName = config.getJdbcDriverClass();
+ url = config.getJdbcUrl();
+ username = config.getJdbcUsername();
+ password = config.getJdbcPassword();
+ maxActive = config.getPoolMaxTotal();
+ maxIdle = config.getPoolMaxIdle();
+ minIdle = config.getPoolMinIdle();
+ }
+}
diff --git a/src/spark-project/sparder/src/main/java/org/apache/kylin/query/pushdown/JdbcPushDownConnectionManager.java b/src/spark-project/sparder/src/main/java/org/apache/kylin/query/pushdown/JdbcPushDownConnectionManager.java
index e4a625f837..b6f3e9528d 100644
--- a/src/spark-project/sparder/src/main/java/org/apache/kylin/query/pushdown/JdbcPushDownConnectionManager.java
+++ b/src/spark-project/sparder/src/main/java/org/apache/kylin/query/pushdown/JdbcPushDownConnectionManager.java
@@ -20,29 +20,43 @@ package org.apache.kylin.query.pushdown;
import java.sql.Connection;
import java.sql.SQLException;
+import java.util.Map;
import org.apache.commons.dbcp.BasicDataSource;
import org.apache.kylin.common.KylinConfig;
+import com.google.common.collect.Maps;
+
public class JdbcPushDownConnectionManager {
- private volatile static JdbcPushDownConnectionManager manager = null;
+ private static final Map<String, JdbcPushDownConnectionManager> managerMap = Maps.newConcurrentMap();
- static JdbcPushDownConnectionManager getConnectionManager(KylinConfig config) throws ClassNotFoundException {
- if (manager == null) {
+ static JdbcPushDownConnectionManager getConnectionManager(KylinConfig config, String project)
+ throws ClassNotFoundException {
+ JdbcPushDownConnectionManager manager = managerMap.get(project);
+ DataSourceConfig newDataSourceConfig = new DataSourceConfig(config);
+ if (needUpdateProjectConnectionManager(manager, newDataSourceConfig)) {
synchronized (JdbcPushDownConnectionManager.class) {
- if (manager == null) {
- manager = new JdbcPushDownConnectionManager(config);
+ if (needUpdateProjectConnectionManager(manager, newDataSourceConfig)) {
+ manager = new JdbcPushDownConnectionManager(config, newDataSourceConfig);
+ managerMap.put(project, manager);
}
}
}
return manager;
}
+ static boolean needUpdateProjectConnectionManager(JdbcPushDownConnectionManager manager, DataSourceConfig config) {
+ return manager == null || !manager.dataSourceConfig.equals(config);
+ }
+
private final BasicDataSource dataSource;
+ private final DataSourceConfig dataSourceConfig;
- private JdbcPushDownConnectionManager(KylinConfig config) throws ClassNotFoundException {
+ private JdbcPushDownConnectionManager(KylinConfig config, DataSourceConfig newDataSourceConfig)
+ throws ClassNotFoundException {
dataSource = new BasicDataSource();
+ dataSourceConfig = newDataSourceConfig;
Class.forName(config.getJdbcDriverClass());
dataSource.setDriverClassName(config.getJdbcDriverClass());
diff --git a/src/spark-project/sparder/src/main/java/org/apache/kylin/query/pushdown/PushDownRunnerJdbcImpl.java b/src/spark-project/sparder/src/main/java/org/apache/kylin/query/pushdown/PushDownRunnerJdbcImpl.java
index c18a024d6f..4b53b720c0 100644
--- a/src/spark-project/sparder/src/main/java/org/apache/kylin/query/pushdown/PushDownRunnerJdbcImpl.java
+++ b/src/spark-project/sparder/src/main/java/org/apache/kylin/query/pushdown/PushDownRunnerJdbcImpl.java
@@ -37,17 +37,22 @@ public class PushDownRunnerJdbcImpl implements IPushDownRunner {
private JdbcPushDownConnectionManager manager = null;
@Override
- public void init(KylinConfig config) {
+ public void init(KylinConfig config, String project) {
try {
- manager = JdbcPushDownConnectionManager.getConnectionManager(config);
+ manager = JdbcPushDownConnectionManager.getConnectionManager(config, project);
} catch (Exception e) {
throw new RuntimeException(e.getMessage(), e);
}
}
+ @Override
+ public void init(KylinConfig config) {
+ init(config, "");
+ }
+
@Override
public void executeQuery(String query, List<List<String>> results, List<SelectedColumnMeta> columnMetas,
- String project) throws Exception {
+ String project) throws Exception {
Statement statement = null;
Connection connection = manager.getConnection();
ResultSet resultSet = null;
diff --git a/src/spark-project/sparder/src/main/java/org/apache/kylin/query/pushdown/PushDownRunnerSparkImpl.java b/src/spark-project/sparder/src/main/java/org/apache/kylin/query/pushdown/PushDownRunnerSparkImpl.java
index be5bcfd908..81887d9c8f 100644
--- a/src/spark-project/sparder/src/main/java/org/apache/kylin/query/pushdown/PushDownRunnerSparkImpl.java
+++ b/src/spark-project/sparder/src/main/java/org/apache/kylin/query/pushdown/PushDownRunnerSparkImpl.java
@@ -23,10 +23,10 @@ import java.util.List;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.QueryContext;
+import org.apache.kylin.metadata.query.StructField;
import org.apache.kylin.metadata.querymeta.SelectedColumnMeta;
import org.apache.kylin.source.adhocquery.IPushDownRunner;
import org.apache.kylin.source.adhocquery.PushdownResult;
-import org.apache.kylin.metadata.query.StructField;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -38,6 +38,11 @@ public class PushDownRunnerSparkImpl implements IPushDownRunner {
// SparkSession has been initialized
}
+ @Override
+ public void init(KylinConfig config, String project) {
+ init(config);
+ }
+
@Override
public void executeQuery(String query, List<List<String>> results, List<SelectedColumnMeta> columnMetas,
String project) {
diff --git a/src/spark-project/sparder/src/test/java/org/apache/kylin/query/pushdown/PushDownRunnerJdbcImplTest.java b/src/spark-project/sparder/src/test/java/org/apache/kylin/query/pushdown/PushDownRunnerJdbcImplTest.java
index 9c99102c50..27fd11e90d 100644
--- a/src/spark-project/sparder/src/test/java/org/apache/kylin/query/pushdown/PushDownRunnerJdbcImplTest.java
+++ b/src/spark-project/sparder/src/test/java/org/apache/kylin/query/pushdown/PushDownRunnerJdbcImplTest.java
@@ -18,26 +18,28 @@
package org.apache.kylin.query.pushdown;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-import lombok.val;
+import java.io.File;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.util.LinkedHashMap;
+import java.util.List;
+
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.KylinConfigExt;
-import org.apache.kylin.metadata.project.ProjectInstance;
-import org.apache.kylin.metadata.querymeta.SelectedColumnMeta;
import org.apache.kylin.common.util.NLocalFileMetadataTestCase;
import org.apache.kylin.common.util.TempMetadataBuilder;
import org.apache.kylin.metadata.project.NProjectManager;
+import org.apache.kylin.metadata.project.ProjectInstance;
+import org.apache.kylin.metadata.querymeta.SelectedColumnMeta;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
-import java.io.File;
-import java.sql.Connection;
-import java.sql.DriverManager;
-import java.util.LinkedHashMap;
-import java.util.List;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+
+import lombok.val;
public class PushDownRunnerJdbcImplTest extends NLocalFileMetadataTestCase {
@@ -122,11 +124,48 @@ public class PushDownRunnerJdbcImplTest extends NLocalFileMetadataTestCase {
npr.updateProject(projectInstance);
KylinConfigExt config = projectInstance.getConfig();
PushDownRunnerJdbcImpl pushDownRunnerJdbc = new PushDownRunnerJdbcImpl();
- pushDownRunnerJdbc.init(config);
+ pushDownRunnerJdbc.init(config, projectInstance.getName());
+ String sql = "select 1";
+ List<List<String>> returnRows = Lists.newArrayList();
+ List<SelectedColumnMeta> returnColumnMeta = Lists.newArrayList();
+ pushDownRunnerJdbc.executeQuery(sql, returnRows, returnColumnMeta, "default");
+ Assert.assertEquals("1", returnRows.get(0).get(0));
+ }
+
+ @Test
+ public void testProjectPushDownJdbc() throws Exception {
+ NProjectManager npr = NProjectManager.getInstance(getTestConfig());
+ ProjectInstance projectInstance = npr.getProject("default");
+ projectInstance.setDefaultDatabase("SSB");
+ LinkedHashMap<String, String> overrideKylinProps = Maps.newLinkedHashMap();
+ overrideKylinProps.put("kylin.query.pushdown.jdbc.url", "jdbc:h2:mem:db_default");
+ overrideKylinProps.put("kylin.query.pushdown.jdbc.driver", "org.h2.Driver");
+ overrideKylinProps.put("kylin.query.pushdown.jdbc.username", "sa");
+ overrideKylinProps.put("kylin.query.pushdown.jdbc.password", "");
+ projectInstance.setOverrideKylinProps(overrideKylinProps);
+ npr.updateProject(projectInstance);
+ KylinConfigExt config = projectInstance.getConfig();
+ PushDownRunnerJdbcImpl pushDownRunnerJdbc = new PushDownRunnerJdbcImpl();
+ pushDownRunnerJdbc.init(config, projectInstance.getName());
String sql = "select 1";
List<List<String>> returnRows = Lists.newArrayList();
List<SelectedColumnMeta> returnColumnMeta = Lists.newArrayList();
pushDownRunnerJdbc.executeQuery(sql, returnRows, returnColumnMeta, "default");
Assert.assertEquals("1", returnRows.get(0).get(0));
+
+ ProjectInstance projectInstance2 = npr.getProject("demo");
+ projectInstance2.setDefaultDatabase("SSB");
+ overrideKylinProps = Maps.newLinkedHashMap();
+ overrideKylinProps.put("kylin.query.pushdown.jdbc.url", "jdbc:h2:mem:db_default");
+ overrideKylinProps.put("kylin.query.pushdown.jdbc.driver", "org.h2.Driver");
+ overrideKylinProps.put("kylin.query.pushdown.jdbc.username", "sa");
+ overrideKylinProps.put("kylin.query.pushdown.jdbc.password", "");
+ projectInstance2.setOverrideKylinProps(overrideKylinProps);
+ npr.updateProject(projectInstance2);
+ config = projectInstance2.getConfig();
+ pushDownRunnerJdbc = new PushDownRunnerJdbcImpl();
+ pushDownRunnerJdbc.init(config, projectInstance2.getName());
+ pushDownRunnerJdbc.executeQuery(sql, returnRows, returnColumnMeta, "demo");
+ Assert.assertEquals("1", returnRows.get(0).get(0));
}
}
diff --git a/src/spark-project/sparder/src/test/java/org/apache/kylin/query/pushdown/PushDownRunnerSparkImplTest.java b/src/spark-project/sparder/src/test/java/org/apache/kylin/query/pushdown/PushDownRunnerSparkImplTest.java
index 46c8a8fa7c..5694f11483 100644
--- a/src/spark-project/sparder/src/test/java/org/apache/kylin/query/pushdown/PushDownRunnerSparkImplTest.java
+++ b/src/spark-project/sparder/src/test/java/org/apache/kylin/query/pushdown/PushDownRunnerSparkImplTest.java
@@ -23,8 +23,8 @@ import java.util.List;
import java.util.UUID;
import org.apache.kylin.common.QueryContext;
-import org.apache.kylin.metadata.querymeta.SelectedColumnMeta;
import org.apache.kylin.common.util.NLocalFileMetadataTestCase;
+import org.apache.kylin.metadata.querymeta.SelectedColumnMeta;
import org.apache.spark.sql.SparderEnv;
import org.apache.spark.sql.SparkSession;
import org.apache.spark.sql.types.DataTypes;
@@ -70,7 +70,7 @@ public class PushDownRunnerSparkImplTest extends NLocalFileMetadataTestCase {
@Test
public void testCast() {
PushDownRunnerSparkImpl pushDownRunnerSpark = new PushDownRunnerSparkImpl();
- pushDownRunnerSpark.init(null);
+ pushDownRunnerSpark.init(null, "tpch");
List<List<String>> returnRows = Lists.newArrayList();
List<SelectedColumnMeta> returnColumnMeta = Lists.newArrayList();
@@ -108,7 +108,7 @@ public class PushDownRunnerSparkImplTest extends NLocalFileMetadataTestCase {
@Test
public void testPushDownRunnerSpark() {
PushDownRunnerSparkImpl pushDownRunnerSpark = new PushDownRunnerSparkImpl();
- pushDownRunnerSpark.init(null);
+ pushDownRunnerSpark.init(null, "tpch");
List<List<String>> returnRows = Lists.newArrayList();
List<SelectedColumnMeta> returnColumnMeta = Lists.newArrayList();
@@ -124,7 +124,7 @@ public class PushDownRunnerSparkImplTest extends NLocalFileMetadataTestCase {
@Test
public void testPushDownRunnerSparkWithDotColumn() {
PushDownRunnerSparkImpl pushDownRunnerSpark = new PushDownRunnerSparkImpl();
- pushDownRunnerSpark.init(null);
+ pushDownRunnerSpark.init(null, "tpch");
List<List<String>> returnRows = Lists.newArrayList();
List<SelectedColumnMeta> returnColumnMeta = Lists.newArrayList();
@@ -140,7 +140,7 @@ public class PushDownRunnerSparkImplTest extends NLocalFileMetadataTestCase {
@Test
public void testSelectTwoSameExpr() {
PushDownRunnerSparkImpl pushDownRunnerSpark = new PushDownRunnerSparkImpl();
- pushDownRunnerSpark.init(null);
+ pushDownRunnerSpark.init(null, "tpch");
List<List<String>> returnRows = Lists.newArrayList();
List<SelectedColumnMeta> returnColumnMeta = Lists.newArrayList();
@@ -156,7 +156,7 @@ public class PushDownRunnerSparkImplTest extends NLocalFileMetadataTestCase {
@Test
public void testCaseSensitiveOnAlias() {
PushDownRunnerSparkImpl pushDownRunnerSpark = new PushDownRunnerSparkImpl();
- pushDownRunnerSpark.init(null);
+ pushDownRunnerSpark.init(null, "tpch");
List<List<String>> returnRows = Lists.newArrayList();
List<SelectedColumnMeta> returnColumnMeta = Lists.newArrayList();
[kylin] 03/17: KYLIN-5390 Build segment support for overlap segments
Posted by xx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit a9de3a2a27169a9695684a7dd1c2f7c5f807e10d
Author: sibingzhang <74...@users.noreply.github.com>
AuthorDate: Wed Nov 9 14:49:15 2022 +0800
KYLIN-5390 Build segment support for overlap segments
---
.../org/apache/kylin/common/KylinConfigBase.java | 4 ++
.../resources/kylin_error_msg_conf_cn.properties | 2 +-
.../resources/kylin_error_msg_conf_en.properties | 2 +-
.../apache/kylin/common/KylinConfigBaseTest.java | 9 +++
.../org/apache/kylin/job/common/SegmentUtil.java | 18 ++---
.../org/apache/kylin/job/common/SegmentsTest.java | 53 ++++++++++++--
.../kylin/rest/controller/SegmentController.java | 3 +-
.../kylin/rest/service/ModelBuildService.java | 18 ++++-
.../apache/kylin/rest/service/ModelService.java | 53 +++++++++-----
.../kylin/rest/service/ModelServiceTest.java | 80 +++++++++++++++++++++-
.../spark/merger/AfterBuildResourceMerger.java | 6 +-
11 files changed, 209 insertions(+), 39 deletions(-)
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 64606df279..309ce19ced 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -3670,6 +3670,10 @@ public abstract class KylinConfigBase implements Serializable {
return Integer.parseInt(getOptional("kylin.second-storage.wait-lock-timeout", "180"));
}
+ public boolean isBuildSegmentOverlapEnabled() {
+ return Boolean.parseBoolean(getOptional("kylin.build.segment-overlap-enabled", FALSE));
+ }
+
public boolean getDDLEnabled(){
return Boolean.parseBoolean(getOptional("kylin.source.ddl.enabled", FALSE));
}
diff --git a/src/core-common/src/main/resources/kylin_error_msg_conf_cn.properties b/src/core-common/src/main/resources/kylin_error_msg_conf_cn.properties
index 759514ee40..7fab056ac7 100644
--- a/src/core-common/src/main/resources/kylin_error_msg_conf_cn.properties
+++ b/src/core-common/src/main/resources/kylin_error_msg_conf_cn.properties
@@ -40,7 +40,7 @@ KE-010025201=无法找到相关 Cube。
## 100222XX Segment
KE-010022201=无法刷新，Segment 范围 “%s” 超出了加载数据的范围 “%s”。请修改后重试。
-KE-010022202=无法构建，待构建的范围和已构建的范围在 “%s” 到 “%s” 之间存在重合。请修改后重试。
+KE-010022202=无法构建，待构建的范围和已构建的范围 “%s” 存在重合。请修改后重试。
KE-010022203=无法刷新，请选择 Segment 后再试。
KE-010022204=无法刷新，部分 Segment 正在构建。请稍后再试。
KE-010022205=无法刷新，所选 Segment 范围为空。请重新选择后再试。
diff --git a/src/core-common/src/main/resources/kylin_error_msg_conf_en.properties b/src/core-common/src/main/resources/kylin_error_msg_conf_en.properties
index 02ecccdb25..967a6c7d8c 100644
--- a/src/core-common/src/main/resources/kylin_error_msg_conf_en.properties
+++ b/src/core-common/src/main/resources/kylin_error_msg_conf_en.properties
@@ -40,7 +40,7 @@ KE-010025201=Can't find the cube.
## 100222XX Segment
KE-010022201=Can't refresh. The segment range "%s" exceeds the range of loaded data, which is "%s". Please modify and try again.
-KE-010022202=Can't build segment. The specified data range overlaps with the built segments from "%s" to "%s". Please modify and try again.
+KE-010022202=Can't build segment. The specified data range overlaps with the built segments "%s". Please modify and try again.
KE-010022203=Can't refresh. Please select segment and try again.
KE-010022204=Can't refresh, some segments are being built. Please try again later.
KE-010022205=Can't refresh, the selected segment range is empty. Please reselect and try again.
diff --git a/src/core-common/src/test/java/org/apache/kylin/common/KylinConfigBaseTest.java b/src/core-common/src/test/java/org/apache/kylin/common/KylinConfigBaseTest.java
index 16f653d46b..f2c3cb1a2f 100644
--- a/src/core-common/src/test/java/org/apache/kylin/common/KylinConfigBaseTest.java
+++ b/src/core-common/src/test/java/org/apache/kylin/common/KylinConfigBaseTest.java
@@ -1380,6 +1380,15 @@ class KylinConfigBaseTest {
val sub2 = config.getSubstitutor();
Assertions.assertSame(sub1, sub2);
}
+
+ @Test
+ void testIsBuildSegmentOverlapEnabled() {
+ KylinConfig config = KylinConfig.getInstanceFromEnv();
+ config.setProperty("kylin.build.segment-overlap-enabled", "false");
+ assertFalse(config.isBuildSegmentOverlapEnabled());
+ config.setProperty("kylin.build.segment-overlap-enabled", "true");
+ assertTrue(config.isBuildSegmentOverlapEnabled());
+ }
}
class EnvironmentUpdateUtils {
diff --git a/src/core-job/src/main/java/org/apache/kylin/job/common/SegmentUtil.java b/src/core-job/src/main/java/org/apache/kylin/job/common/SegmentUtil.java
index e04777ef7f..0185cee866 100644
--- a/src/core-job/src/main/java/org/apache/kylin/job/common/SegmentUtil.java
+++ b/src/core-job/src/main/java/org/apache/kylin/job/common/SegmentUtil.java
@@ -17,6 +17,7 @@
*/
package org.apache.kylin.job.common;
+import static org.apache.kylin.job.execution.JobTypeEnum.INC_BUILD;
import static org.apache.kylin.job.execution.JobTypeEnum.INDEX_BUILD;
import static org.apache.kylin.job.execution.JobTypeEnum.SUB_PARTITION_BUILD;
@@ -30,13 +31,13 @@ import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.job.execution.AbstractExecutable;
import org.apache.kylin.job.execution.ExecutableState;
import org.apache.kylin.job.execution.NExecutableManager;
+import org.apache.kylin.metadata.cube.model.NDataSegment;
+import org.apache.kylin.metadata.cube.model.NDataflowManager;
+import org.apache.kylin.metadata.cube.model.PartitionStatusEnum;
import org.apache.kylin.metadata.model.ISegment;
import org.apache.kylin.metadata.model.SegmentStatusEnum;
import org.apache.kylin.metadata.model.SegmentStatusEnumToDisplay;
import org.apache.kylin.metadata.model.Segments;
-import org.apache.kylin.metadata.cube.model.NDataSegment;
-import org.apache.kylin.metadata.cube.model.NDataflowManager;
-import org.apache.kylin.metadata.cube.model.PartitionStatusEnum;
import com.google.common.base.Preconditions;
import com.google.common.collect.Sets;
@@ -64,12 +65,13 @@ public class SegmentUtil {
Segments<T> overlapSegs = segments.getSegmentsByRange(segment.getSegRange());
overlapSegs.remove(segment);
if (SegmentStatusEnum.NEW == segment.getStatus()) {
- if (CollectionUtils.isEmpty(overlapSegs)) {
- return SegmentStatusEnumToDisplay.LOADING;
+ if (!CollectionUtils.isEmpty(overlapSegs)
+ && overlapSegs.get(0).getSegRange().entireOverlaps(segment.getSegRange())) {
+ return SegmentStatusEnumToDisplay.REFRESHING;
}
- if (overlapSegs.get(0).getSegRange().entireOverlaps(segment.getSegRange())) {
- return SegmentStatusEnumToDisplay.REFRESHING;
+ if (CollectionUtils.isEmpty(overlapSegs) || anyIndexJobRunning(segment, executables)) {
+ return SegmentStatusEnumToDisplay.LOADING;
}
return SegmentStatusEnumToDisplay.MERGING;
@@ -102,7 +104,7 @@ public class SegmentUtil {
protected static <T extends ISegment> boolean anyIndexJobRunning(T segment) {
KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
NExecutableManager execManager = NExecutableManager.getInstance(kylinConfig, segment.getModel().getProject());
- val executables = execManager.listExecByJobTypeAndStatus(ExecutableState::isRunning, INDEX_BUILD,
+ val executables = execManager.listExecByJobTypeAndStatus(ExecutableState::isRunning, INDEX_BUILD, INC_BUILD,
SUB_PARTITION_BUILD);
return executables.stream().anyMatch(task -> task.getSegmentIds().contains(segment.getId()));
}
diff --git a/src/core-job/src/test/java/org/apache/kylin/job/common/SegmentsTest.java b/src/core-job/src/test/java/org/apache/kylin/job/common/SegmentsTest.java
index 97eab4cdb8..587e688e21 100644
--- a/src/core-job/src/test/java/org/apache/kylin/job/common/SegmentsTest.java
+++ b/src/core-job/src/test/java/org/apache/kylin/job/common/SegmentsTest.java
@@ -19,19 +19,23 @@
package org.apache.kylin.job.common;
import org.apache.kylin.common.util.RandomUtil;
+import org.apache.kylin.job.execution.DefaultExecutable;
+import org.apache.kylin.junit.TimeZoneTestRunner;
+import org.apache.kylin.metadata.cube.model.NBatchConstants;
+import org.apache.kylin.metadata.cube.model.NDataSegment;
+import org.apache.kylin.metadata.cube.model.NDataflow;
import org.apache.kylin.metadata.model.SegmentRange;
import org.apache.kylin.metadata.model.SegmentStatusEnum;
import org.apache.kylin.metadata.model.SegmentStatusEnumToDisplay;
import org.apache.kylin.metadata.model.Segments;
-import org.apache.kylin.junit.TimeZoneTestRunner;
-import org.apache.kylin.metadata.cube.model.NDataSegment;
-import org.apache.kylin.metadata.cube.model.NDataflow;
import org.junit.After;
import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Mockito;
+import com.google.common.collect.Lists;
+
import lombok.val;
@RunWith(TimeZoneTestRunner.class)
@@ -59,6 +63,41 @@ public class SegmentsTest {
Assert.assertEquals(status, SegmentStatusEnumToDisplay.LOADING);
}
+ @Test
+ public void testGetSegmentStatusToDisplay_Loading() {
+ Segments segments = new Segments();
+ val seg = NDataSegment.empty();
+ seg.setId("1");
+ seg.setSegmentRange(new SegmentRange.TimePartitionedSegmentRange(0L, 10L));
+ seg.setStatus(SegmentStatusEnum.READY);
+ segments.add(seg);
+
+ val seg2 = NDataSegment.empty();
+ seg2.setId("2");
+ seg2.setSegmentRange(new SegmentRange.TimePartitionedSegmentRange(0L, 15L));
+ seg2.setStatus(SegmentStatusEnum.NEW);
+
+ val job = new DefaultExecutable();
+ job.setParam(NBatchConstants.P_SEGMENT_IDS, "2");
+
+ SegmentStatusEnumToDisplay status = SegmentUtil.getSegmentStatusToDisplay(segments, seg2,
+ Lists.newArrayList(job));
+ Assert.assertEquals(SegmentStatusEnumToDisplay.LOADING, status);
+
+ val seg3 = NDataSegment.empty();
+ seg3.setId("3");
+ seg3.setSegmentRange(new SegmentRange.TimePartitionedSegmentRange(11L, 12L));
+ seg3.setStatus(SegmentStatusEnum.NEW);
+ segments.add(seg3);
+
+ val job2 = new DefaultExecutable();
+ job2.setParam(NBatchConstants.P_SEGMENT_IDS, "3");
+
+ SegmentStatusEnumToDisplay status2 = SegmentUtil.getSegmentStatusToDisplay(segments, seg3,
+ Lists.newArrayList(job2));
+ Assert.assertEquals(SegmentStatusEnumToDisplay.LOADING, status2);
+ }
+
@Test
public void testGetSegmentStatusToDisplay_Ready() {
Segments segments = new Segments();
@@ -157,15 +196,21 @@ public class SegmentsTest {
newSeg.setStatus(SegmentStatusEnum.NEW);
segments.add(newSeg);
+ Mockito.mockStatic(SegmentUtil.class);
+ Mockito.when(SegmentUtil.getSegmentStatusToDisplay(segments, newSeg, null)).thenCallRealMethod();
+ Mockito.when(SegmentUtil.anyIndexJobRunning(newSeg)).thenReturn(false);
SegmentStatusEnumToDisplay status = SegmentUtil.getSegmentStatusToDisplay(segments, newSeg, null);
Assert.assertEquals(status, SegmentStatusEnumToDisplay.MERGING);
+ Mockito.when(SegmentUtil.getSegmentStatusToDisplay(segments, seg, null)).thenCallRealMethod();
+ Mockito.when(SegmentUtil.anyIndexJobRunning(seg)).thenReturn(false);
SegmentStatusEnumToDisplay status2 = SegmentUtil.getSegmentStatusToDisplay(segments, seg, null);
Assert.assertEquals(status2, SegmentStatusEnumToDisplay.LOCKED);
+ Mockito.when(SegmentUtil.getSegmentStatusToDisplay(segments, seg2, null)).thenCallRealMethod();
+ Mockito.when(SegmentUtil.anyIndexJobRunning(seg2)).thenReturn(false);
SegmentStatusEnumToDisplay status3 = SegmentUtil.getSegmentStatusToDisplay(segments, seg2, null);
Assert.assertEquals(status3, SegmentStatusEnumToDisplay.LOCKED);
-
}
public NDataSegment newReadySegment(Long startTime, Long endTime) {
diff --git a/src/data-loading-server/src/main/java/org/apache/kylin/rest/controller/SegmentController.java b/src/data-loading-server/src/main/java/org/apache/kylin/rest/controller/SegmentController.java
index 5f722bd282..d1f706e275 100644
--- a/src/data-loading-server/src/main/java/org/apache/kylin/rest/controller/SegmentController.java
+++ b/src/data-loading-server/src/main/java/org/apache/kylin/rest/controller/SegmentController.java
@@ -166,7 +166,8 @@ public class SegmentController extends BaseController {
modelId);
validateDataRange(buildSegmentsRequest.getStart(), buildSegmentsRequest.getEnd(), partitionColumnFormat);
val res = modelService.checkSegHoleExistIfNewRangeBuild(buildSegmentsRequest.getProject(), modelId,
- buildSegmentsRequest.getStart(), buildSegmentsRequest.getEnd());
+ buildSegmentsRequest.getStart(), buildSegmentsRequest.getEnd(),
+ buildSegmentsRequest.isBuildAllIndexes(), buildSegmentsRequest.getBatchIndexIds());
return new EnvelopeResponse<>(KylinException.CODE_SUCCESS, res, "");
}
diff --git a/src/data-loading-service/src/main/java/org/apache/kylin/rest/service/ModelBuildService.java b/src/data-loading-service/src/main/java/org/apache/kylin/rest/service/ModelBuildService.java
index 93724c8c26..6d4e17b81b 100644
--- a/src/data-loading-service/src/main/java/org/apache/kylin/rest/service/ModelBuildService.java
+++ b/src/data-loading-service/src/main/java/org/apache/kylin/rest/service/ModelBuildService.java
@@ -25,6 +25,7 @@ import static org.apache.kylin.common.exception.code.ErrorCodeServer.JOB_CONCURR
import static org.apache.kylin.common.exception.code.ErrorCodeServer.JOB_CREATE_CHECK_MULTI_PARTITION_ABANDON;
import static org.apache.kylin.common.exception.code.ErrorCodeServer.JOB_CREATE_CHECK_MULTI_PARTITION_DUPLICATE;
import static org.apache.kylin.common.exception.code.ErrorCodeServer.JOB_CREATE_CHECK_MULTI_PARTITION_EMPTY;
+import static org.apache.kylin.common.exception.code.ErrorCodeServer.SEGMENT_BUILD_RANGE_OVERLAP;
import java.io.IOException;
import java.util.ArrayList;
@@ -34,6 +35,7 @@ import java.util.List;
import java.util.Locale;
import java.util.Objects;
import java.util.Set;
+import java.util.StringJoiner;
import java.util.stream.Collectors;
import org.apache.commons.collections.CollectionUtils;
@@ -371,7 +373,9 @@ public class ModelBuildService extends AbstractModelService implements ModelBuil
Preconditions.checkArgument(!PushDownUtil.needPushdown(params.getStart(), params.getEnd()),
"Load data must set start and end date");
val segmentRangeToBuild = SourceFactory.getSource(table).getSegmentRange(params.getStart(), params.getEnd());
- modelService.checkSegmentToBuildOverlapsBuilt(project, modelId, segmentRangeToBuild);
+ List<NDataSegment> overlapSegments = modelService.checkSegmentToBuildOverlapsBuilt(project,
+ modelDescInTransaction, segmentRangeToBuild, params.isNeedBuild(), params.getBatchIndexIds());
+ buildSegmentOverlapExceptionInfo(overlapSegments);
modelService.saveDateFormatIfNotExist(project, modelId, params.getPartitionColFormat());
checkMultiPartitionBuildParam(modelDescInTransaction, params);
NDataSegment newSegment = getManager(NDataflowManager.class, project).appendSegment(df, segmentRangeToBuild,
@@ -394,6 +398,18 @@ public class ModelBuildService extends AbstractModelService implements ModelBuil
.licenseCheckWrap(project, () -> jobManager.addSegmentJob(jobParam)));
}
+ private void buildSegmentOverlapExceptionInfo(List<NDataSegment> overlapSegments) {
+ if (CollectionUtils.isEmpty(overlapSegments)) {
+ return;
+ }
+
+ StringJoiner joiner = new StringJoiner(",", "[", "]");
+ for (NDataSegment seg : overlapSegments) {
+ joiner.add(seg.getName());
+ }
+ throw new KylinException(SEGMENT_BUILD_RANGE_OVERLAP, joiner.toString());
+ }
+
public void checkMultiPartitionBuildParam(NDataModel model, IncrementBuildSegmentParams params) {
if (!model.isMultiPartitionModel()) {
return;
diff --git a/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelService.java b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelService.java
index 07cab15306..e4af48b957 100644
--- a/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelService.java
+++ b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelService.java
@@ -57,7 +57,6 @@ import static org.apache.kylin.common.exception.code.ErrorCodeServer.MODEL_NAME_
import static org.apache.kylin.common.exception.code.ErrorCodeServer.MODEL_NAME_NOT_EXIST;
import static org.apache.kylin.common.exception.code.ErrorCodeServer.PARAMETER_INVALID_SUPPORT_LIST;
import static org.apache.kylin.common.exception.code.ErrorCodeServer.PROJECT_NOT_EXIST;
-import static org.apache.kylin.common.exception.code.ErrorCodeServer.SEGMENT_BUILD_RANGE_OVERLAP;
import static org.apache.kylin.common.exception.code.ErrorCodeServer.SEGMENT_LOCKED;
import static org.apache.kylin.common.exception.code.ErrorCodeServer.SEGMENT_MERGE_CONTAINS_GAPS;
import static org.apache.kylin.common.exception.code.ErrorCodeServer.SEGMENT_NOT_EXIST_ID;
@@ -929,7 +928,7 @@ public class ModelService extends AbstractModelService implements TableModelSupp
KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
NExecutableManager execManager = NExecutableManager.getInstance(kylinConfig, project);
return execManager.listPartialExec(path -> StringUtils.endsWith(path, modelId), ExecutableState::isRunning,
- INDEX_BUILD, JobTypeEnum.SUB_PARTITION_BUILD);
+ INDEX_BUILD, INC_BUILD, JobTypeEnum.SUB_PARTITION_BUILD);
}
public List<NDataSegmentResponse> getSegmentsResponse(String modelId, String project, String start, String end,
@@ -2295,7 +2294,7 @@ public class ModelService extends AbstractModelService implements TableModelSupp
}
public SegmentCheckResponse checkSegHoleExistIfNewRangeBuild(String project, String modelId, String start,
- String end) {
+ String end, boolean isBuildAllIndexes, List<Long> batchIndexIds) {
aclEvaluate.checkProjectOperationPermission(project);
Preconditions.checkArgument(!PushDownUtil.needPushdown(start, end), "Load data must set start and end date");
NDataModel dataModelDesc = getManager(NDataModelManager.class, project).getDataModelDesc(modelId);
@@ -2304,18 +2303,17 @@ public class ModelService extends AbstractModelService implements TableModelSupp
SegmentRange segmentRangeToBuild = SourceFactory.getSource(table).getSegmentRange(start, end);
List<NDataSegment> segmentGaps = NDataflowManager.getInstance(KylinConfig.getInstanceFromEnv(), project)
.checkHoleIfNewSegBuild(modelId, segmentRangeToBuild);
-
- Segments<NDataSegment> segments = getSegmentsByRange(modelId, project, "0", "" + Long.MAX_VALUE);
- val overlapSegments = segments.stream().filter(seg -> seg.getSegRange().overlaps(segmentRangeToBuild))
- .map(seg -> new SegmentRangeResponse(seg.getTSRange().getStart(), seg.getTSRange().getEnd()))
+ List<NDataSegment> overlapSegments = checkSegmentToBuildOverlapsBuilt(project, dataModelDesc,
+ segmentRangeToBuild, isBuildAllIndexes, batchIndexIds);
+ val overlapSegmentResponses = overlapSegments.stream().map(
+ segment -> new SegmentRangeResponse(segment.getTSRange().getStart(), segment.getTSRange().getEnd()))
.collect(Collectors.toList());
-
SegmentCheckResponse segmentCheckResponse = new SegmentCheckResponse();
val segHoles = segmentGaps.stream()
.map(seg -> new SegmentRangeResponse(seg.getTSRange().getStart(), seg.getTSRange().getEnd()))
.collect(Collectors.toList());
segmentCheckResponse.setSegmentHoles(segHoles);
- segmentCheckResponse.setOverlapSegments(overlapSegments);
+ segmentCheckResponse.setOverlapSegments(overlapSegmentResponses);
return segmentCheckResponse;
}
@@ -2563,17 +2561,36 @@ public class ModelService extends AbstractModelService implements TableModelSupp
datamodelManager.updateDataModelDesc(modelUpdate);
}
- public void checkSegmentToBuildOverlapsBuilt(String project, String model, SegmentRange segmentRangeToBuild) {
- Segments<NDataSegment> segments = getSegmentsByRange(model, project, "0", "" + Long.MAX_VALUE);
- if (!CollectionUtils.isEmpty(segments)) {
- for (NDataSegment existedSegment : segments) {
- if (existedSegment.getSegRange().overlaps(segmentRangeToBuild)) {
- throw new KylinException(SEGMENT_BUILD_RANGE_OVERLAP,
- existedSegment.getSegRange().getStart().toString(),
- existedSegment.getSegRange().getEnd().toString());
- }
+ public List<NDataSegment> checkSegmentToBuildOverlapsBuilt(String project, NDataModel model,
+ SegmentRange<Long> segmentRangeToBuild, boolean isBuildAllIndexes, List<Long> batchIndexIds) {
+ boolean isOverlap;
+ Segments<NDataSegment> segments = getSegmentsByRange(model.getId(), project, "0", "" + Long.MAX_VALUE);
+ List<NDataSegment> overlapsBuiltSegment = Lists.newArrayListWithCapacity(segments.size());
+
+ if (CollectionUtils.isEmpty(segments)) {
+ return overlapsBuiltSegment;
+ }
+
+ boolean buildSegmentOverlapEnable = getIndexPlan(model.getId(), project).getConfig()
+ .isBuildSegmentOverlapEnabled();
+ boolean isBuildAllIndexesFinally = batchIndexIds == null || batchIndexIds.size() == 0
+ || batchIndexIds.size() == getIndexPlan(model.getId(), project).getAllIndexes().size();
+
+ for (NDataSegment existedSegment : segments) {
+ if (buildSegmentOverlapEnable && NDataModel.ModelType.BATCH == model.getModelType()
+ && !model.isMultiPartitionModel() && isBuildAllIndexes && isBuildAllIndexesFinally
+ && !SecondStorageUtil.isModelEnable(project, model.getId())) {
+ isOverlap = existedSegment.getSegRange().overlaps(segmentRangeToBuild)
+ && !segmentRangeToBuild.contains(existedSegment.getSegRange());
+ } else {
+ isOverlap = existedSegment.getSegRange().overlaps(segmentRangeToBuild);
+ }
+ if (isOverlap) {
+ overlapsBuiltSegment.add(existedSegment);
}
}
+
+ return overlapsBuiltSegment;
}
public ComputedColumnUsageResponse getComputedColumnUsages(String project) {
diff --git a/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelServiceTest.java b/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelServiceTest.java
index e9dc0f22c3..8d3a4462b2 100644
--- a/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelServiceTest.java
+++ b/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelServiceTest.java
@@ -3889,19 +3889,93 @@ public class ModelServiceTest extends SourceTestCase {
Assert.assertEquals(0, res.getOverlapSegments().size());
Assert.assertEquals(1, res.getSegmentHoles().size());
- res = modelService.checkSegHoleExistIfNewRangeBuild(getProject(), modelId, "20000", "30000");
+ res = modelService.checkSegHoleExistIfNewRangeBuild(getProject(), modelId, "20000", "30000", true, null);
Assert.assertEquals(0, res.getOverlapSegments().size());
Assert.assertEquals(3, res.getSegmentHoles().size());
- res = modelService.checkSegHoleExistIfNewRangeBuild(getProject(), modelId, "1", "10");
+ res = modelService.checkSegHoleExistIfNewRangeBuild(getProject(), modelId, "1", "10", true, null);
Assert.assertEquals(0, res.getOverlapSegments().size());
Assert.assertEquals(1, res.getSegmentHoles().size());
- res = modelService.checkSegHoleExistIfNewRangeBuild(getProject(), modelId, "1", "5");
+ res = modelService.checkSegHoleExistIfNewRangeBuild(getProject(), modelId, "1", "5", true, null);
Assert.assertEquals(0, res.getOverlapSegments().size());
Assert.assertEquals(2, res.getSegmentHoles().size());
}
+ @Test
+ public void testCheckSegmentToBuildOverlapsBuilt() throws IOException {
+ KylinConfig kylinConfig = getTestConfig();
+ final String defaultProject = getProject();
+ final String streamingProject = "streaming_test";
+ NDataModelManager modelManager = NDataModelManager.getInstance(kylinConfig, defaultProject);
+
+ kylinConfig.setProperty("kylin.build.segment-overlap-enabled", "true");
+
+ List<NDataSegment> overlapSegments = modelService.checkSegmentToBuildOverlapsBuilt(defaultProject,
+ modelManager.getDataModelDesc("b780e4e4-69af-449e-b09f-05c90dfa04b6"),
+ new SegmentRange.TimePartitionedSegmentRange(1604188800000L, 1604361600000L), true, null);
+ Assert.assertEquals(3, overlapSegments.size());
+
+ val streamingModelManager = NDataModelManager.getInstance(getTestConfig(), streamingProject);
+ List<NDataSegment> overlapSegments2 = modelService.checkSegmentToBuildOverlapsBuilt(streamingProject,
+ streamingModelManager.getDataModelDesc("e78a89dd-847f-4574-8afa-8768b4228b74"),
+ new SegmentRange.KafkaOffsetPartitionedSegmentRange(1613957110000L, 1613957130000L), true, null);
+ Assert.assertEquals(2, overlapSegments2.size());
+
+ String modelId = "abe3bf1a-c4bc-458d-8278-7ea8b00f5e96";
+ NDataModel dataModelDesc = modelManager.getDataModelDesc(modelId);
+ List<NDataSegment> overlapSegments3 = modelService.checkSegmentToBuildOverlapsBuilt(defaultProject,
+ dataModelDesc, new SegmentRange.TimePartitionedSegmentRange(1309891513770L, 1509891513770L), true,
+ null);
+ Assert.assertEquals(0, overlapSegments3.size());
+
+ List<NDataSegment> overlapSegments4 = modelService.checkSegmentToBuildOverlapsBuilt(defaultProject,
+ dataModelDesc, new SegmentRange.TimePartitionedSegmentRange(1309891513770L, 1609891513770L), true,
+ null);
+ Assert.assertEquals(0, overlapSegments4.size());
+
+ List<NDataSegment> overlapSegments5 = modelService.checkSegmentToBuildOverlapsBuilt(defaultProject,
+ dataModelDesc, new SegmentRange.TimePartitionedSegmentRange(1309891513770L, 1609891513770L), false,
+ null);
+ Assert.assertEquals(1, overlapSegments5.size());
+
+ List<NDataSegment> overlapSegments6 = modelService.checkSegmentToBuildOverlapsBuilt(defaultProject,
+ dataModelDesc, new SegmentRange.TimePartitionedSegmentRange(1309891513770L, 1609891513770L), true,
+ Lists.newArrayList());
+ Assert.assertEquals(0, overlapSegments6.size());
+
+ List<NDataSegment> overlapSegments7 = modelService.checkSegmentToBuildOverlapsBuilt(defaultProject,
+ dataModelDesc, new SegmentRange.TimePartitionedSegmentRange(1309891513770L, 1609891513770L), true,
+ Lists.newArrayList(10000L));
+ Assert.assertEquals(1, overlapSegments7.size());
+
+ List<NDataSegment> overlapSegments8 = modelService.checkSegmentToBuildOverlapsBuilt(defaultProject,
+ dataModelDesc, new SegmentRange.TimePartitionedSegmentRange(1309891513780L, 1509891513760L), true,
+ null);
+ Assert.assertEquals(1, overlapSegments8.size());
+
+ MockSecondStorage.mock(defaultProject, new ArrayList<>(), this);
+ val indexPlanManager = NIndexPlanManager.getInstance(KylinConfig.getInstanceFromEnv(), defaultProject);
+ EnhancedUnitOfWork.doInTransactionWithCheckAndRetry(() -> {
+ indexPlanManager.updateIndexPlan(modelId, indexPlan -> {
+ indexPlan.createAndAddBaseIndex(indexPlan.getModel());
+ });
+ return null;
+ }, defaultProject);
+ SecondStorageUtil.initModelMetaData(defaultProject, modelId);
+ Assert.assertTrue(SecondStorageUtil.isModelEnable(defaultProject, modelId));
+ List<NDataSegment> overlapSegments31 = modelService.checkSegmentToBuildOverlapsBuilt(defaultProject,
+ dataModelDesc, new SegmentRange.TimePartitionedSegmentRange(1309891513770L, 1509891513770L), true,
+ null);
+ Assert.assertEquals(1, overlapSegments31.size());
+
+ kylinConfig.setProperty("kylin.build.segment-overlap-enabled", "false");
+ List<NDataSegment> overlapSegments9 = modelService.checkSegmentToBuildOverlapsBuilt(defaultProject,
+ dataModelDesc, new SegmentRange.TimePartitionedSegmentRange(1309891513770L, 1509891513770L), true,
+ null);
+ Assert.assertEquals(1, overlapSegments9.size());
+ }
+
@Test
public void testUpdateModelOwner() throws IOException {
String project = "default";
diff --git a/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/merger/AfterBuildResourceMerger.java b/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/merger/AfterBuildResourceMerger.java
index 1913f45ba2..117d381df8 100644
--- a/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/merger/AfterBuildResourceMerger.java
+++ b/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/merger/AfterBuildResourceMerger.java
@@ -25,16 +25,16 @@ import java.util.Set;
import org.apache.commons.collections.CollectionUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.ResourceStore;
+import org.apache.kylin.engine.spark.ExecutableUtils;
import org.apache.kylin.job.execution.AbstractExecutable;
import org.apache.kylin.job.execution.JobTypeEnum;
-import org.apache.kylin.metadata.model.SegmentStatusEnum;
-import org.apache.kylin.engine.spark.ExecutableUtils;
import org.apache.kylin.metadata.cube.model.NDataLayout;
import org.apache.kylin.metadata.cube.model.NDataSegment;
import org.apache.kylin.metadata.cube.model.NDataflow;
import org.apache.kylin.metadata.cube.model.NDataflowManager;
import org.apache.kylin.metadata.cube.model.NDataflowUpdate;
import org.apache.kylin.metadata.cube.model.PartitionStatusEnum;
+import org.apache.kylin.metadata.model.SegmentStatusEnum;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
@@ -91,6 +91,7 @@ public class AfterBuildResourceMerger extends SparkJobMetadataMerger {
val dfUpdate = new NDataflowUpdate(flowName);
val theSeg = remoteDataflow.getSegment(segmentId);
+ val toRemoveSegments = remoteDataflowManager.getToRemoveSegs(remoteDataflow, theSeg);
if (theSeg.getModel().isMultiPartitionModel()) {
final long lastBuildTime = System.currentTimeMillis();
@@ -107,6 +108,7 @@ public class AfterBuildResourceMerger extends SparkJobMetadataMerger {
theSeg.setStatus(SegmentStatusEnum.READY);
dfUpdate.setToUpdateSegs(theSeg);
+ dfUpdate.setToRemoveSegs(toRemoveSegments.toArray(new NDataSegment[toRemoveSegments.size()]));
dfUpdate.setToAddOrUpdateLayouts(theSeg.getSegDetails().getLayouts().toArray(new NDataLayout[0]));
localDataflowManager.updateDataflow(dfUpdate);
[kylin] 02/17: KYLIN-5389 fix duplicate CC when querying
Posted by xx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit a410bd0c08f3d6f130e52f9dba581e52486998b9
Author: Dorris Zhang <ru...@kyligence.io>
AuthorDate: Mon Nov 7 15:26:18 2022 +0800
KYLIN-5389 fix duplicate CC when querying
---
.../src/main/java/org/apache/kylin/query/schema/OLAPTable.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/query-common/src/main/java/org/apache/kylin/query/schema/OLAPTable.java b/src/query-common/src/main/java/org/apache/kylin/query/schema/OLAPTable.java
index 9551f54bd0..7aa18f093b 100644
--- a/src/query-common/src/main/java/org/apache/kylin/query/schema/OLAPTable.java
+++ b/src/query-common/src/main/java/org/apache/kylin/query/schema/OLAPTable.java
@@ -289,7 +289,7 @@ public class OLAPTable extends AbstractQueryableTable implements TranslatableTab
allColumns.addAll(Lists.newArrayList(ccAsColumnDesc));
}
- return allColumns;
+ return allColumns.stream().distinct().collect(Collectors.toList());
}
// since computed columns are either of different expr and different names,
[kylin] 06/17: KYLIN-5393 fix bug when double quote in pushdown sql
Posted by xx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit bbcf3df8d7e1eafa0ccc2914e76ff8ac122d0215
Author: songzhxlh-max <59...@users.noreply.github.com>
AuthorDate: Thu Nov 17 14:17:23 2022 +0800
KYLIN-5393 fix bug when double quote in pushdown sql
---
.../org/apache/kylin/query/util/QueryUtil.java | 29 ++++++++++++++++++++++
.../org/apache/kylin/query/util/QueryUtilTest.java | 18 ++++++++++++++
2 files changed, 47 insertions(+)
diff --git a/src/query-common/src/main/java/org/apache/kylin/query/util/QueryUtil.java b/src/query-common/src/main/java/org/apache/kylin/query/util/QueryUtil.java
index ed30dc216c..0429f8a8fa 100644
--- a/src/query-common/src/main/java/org/apache/kylin/query/util/QueryUtil.java
+++ b/src/query-common/src/main/java/org/apache/kylin/query/util/QueryUtil.java
@@ -510,9 +510,38 @@ public class QueryUtil {
"Current step: Massage push-down sql. ");
sql = converter.convert(sql, queryParams.getProject(), queryParams.getDefaultSchema());
}
+
+ sql = replaceDoubleQuoteToSingle(sql);
return sql;
}
+ // To keep the results of cube and pushDown are same, we need to replace here
+ public static String replaceDoubleQuoteToSingle(String originSql) {
+ boolean inStrVal = false;
+ boolean needTransfer = false;
+ char[] res = originSql.toCharArray();
+ for (int i = 0; i < res.length; i++) {
+ if (res[i] == '\'') {
+ if (inStrVal) {
+ if (needTransfer) {
+ res[i - 1] = '\\';
+ needTransfer = false;
+ } else {
+ needTransfer = true;
+ }
+ } else {
+ inStrVal = true;
+ }
+ } else {
+ if (needTransfer) {
+ inStrVal = false;
+ needTransfer = false;
+ }
+ }
+ }
+ return new String(res);
+ }
+
static void initPushDownConvertersIfNeeded(KylinConfig kylinConfig) {
String[] currentConverters = pushDownConverters.stream().map(Object::getClass).map(Class::getCanonicalName)
.toArray(String[]::new);
diff --git a/src/query/src/test/java/org/apache/kylin/query/util/QueryUtilTest.java b/src/query/src/test/java/org/apache/kylin/query/util/QueryUtilTest.java
index 1496a92d72..e5084d71ac 100644
--- a/src/query/src/test/java/org/apache/kylin/query/util/QueryUtilTest.java
+++ b/src/query/src/test/java/org/apache/kylin/query/util/QueryUtilTest.java
@@ -196,6 +196,17 @@ public class QueryUtilTest extends NLocalFileMetadataTestCase {
}
}
+ @Test
+ public void testMassagePushDownSqlWithDoubleQuote() {
+ KylinConfig config = KylinConfig.createKylinConfig(new Properties());
+ String sql = "select '''',trans_id from test_kylin_fact where LSTG_FORMAT_NAME like '%''%' group by trans_id limit 2;";
+ QueryParams queryParams = new QueryParams("", sql, "default", false);
+ queryParams.setKylinConfig(config);
+ String massagedSql = QueryUtil.massagePushDownSql(queryParams);
+ String expectedSql = "select '\\'', `TRANS_ID` from `TEST_KYLIN_FACT` where `LSTG_FORMAT_NAME` like '%\\'%' group by `TRANS_ID` limit 2";
+ Assert.assertEquals(expectedSql, massagedSql);
+ }
+
@Test
public void testInit() {
KylinConfig config = KylinConfig.createKylinConfig(new Properties());
@@ -580,4 +591,11 @@ public class QueryUtilTest extends NLocalFileMetadataTestCase {
"select TRANS_ID as test_limit, ORDER_ID as test_offset from TEST_KYLIN_FACT group by TRANS_ID, ORDER_ID",
targetSQL);
}
+
+ @Test
+ public void testReplaceDoubleQuoteToSingle() {
+ String sql = "select ab from table where aa = '' and bb = '''as''n'''";
+ String resSql = "select ab from table where aa = '' and bb = '\\'as\\'n\\''";
+ Assert.assertEquals(resSql, QueryUtil.replaceDoubleQuoteToSingle(sql));
+ }
}
[kylin] 05/17: KYLIN-5392 slow query can not cancel
Posted by xx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 4d3f892b27755ce35015a24fb7ac349fe88495e7
Author: Pengfei Zhan <pe...@kyligence.io>
AuthorDate: Sat Oct 22 01:11:37 2022 +0800
KYLIN-5392 slow query can not cancel
1. fix some case of slow query can not cancel;
2. move KapQueryUtil to QueryUtil;
3. fix some bug of QueryUtil;
4. move QueryUtil#getKylinConfig to NProjectManager#getProjectConfig;
---
.../org/apache/kylin/common/KylinConfigBase.java | 4 +-
.../org/apache/kylin/metadata/model/TableDesc.java | 11 +
.../kylin/metadata/project/NProjectManager.java | 6 +
.../kylin/rest/service/ModelServiceBuildTest.java | 4 +-
.../kylin/newten/NBadQueryAndPushDownTest.java | 5 +-
.../apache/kylin/newten/SlowQueryDetectorTest.java | 13 +-
.../java/org/apache/kylin/query/KylinTestBase.java | 4 +-
.../kylin/query/rules/CalciteRuleTestBase.java | 6 +-
.../java/org/apache/kylin/util/ExecAndComp.java | 16 +-
.../kylin/rest/service/ModelSemanticHelper.java | 6 +-
.../apache/kylin/rest/service/ModelService.java | 15 +-
.../kylin/rest/service/ModelServiceTest.java | 4 +-
.../org/apache/kylin/query/IQueryTransformer.java | 23 ++
.../org/apache/kylin/query/SlowQueryDetector.java | 2 +-
.../security/HackSelectStarWithColumnACL.java | 12 +-
.../org/apache/kylin/query/security/RowFilter.java | 6 +-
.../kylin/query/util/ConvertToComputedColumn.java | 24 +-
.../query/util/DateNumberFilterTransformer.java | 11 +-
.../kylin/query/util/DefaultQueryTransformer.java | 3 +-
.../apache/kylin/query/util/EscapeTransformer.java | 6 +-
.../kylin/query/util/KeywordDefaultDirtyHack.java | 3 +-
.../apache/kylin/query/util/PowerBIConverter.java | 6 +-
.../org/apache/kylin/query/util/PushDownUtil.java | 6 +-
.../apache/kylin/query/util/QueryAliasMatcher.java | 71 +++---
.../util/{KapQueryUtil.java => QueryUtil.java} | 268 +++++++++++++--------
.../query/util/RestoreFromComputedColumn.java | 74 +++---
.../query/util/WithToSubQueryTransformer.java | 6 +-
.../rest/controller/NQueryControllerTest.java | 75 +++---
.../kylin/rest/service/QueryCacheManager.java | 5 +-
.../apache/kylin/rest/service/QueryService.java | 120 ++++-----
.../org/apache/kylin/rest/util/QueryUtils.java | 17 +-
.../rest/metrics/QueryMetricsContextTest.java | 15 +-
.../kylin/rest/service/ModelServiceQueryTest.java | 8 +-
.../kylin/rest/service/QueryServiceTest.java | 36 +--
.../optrule/AbstractAggCaseWhenFunctionRule.java | 4 +-
.../optrule/CountDistinctCaseWhenFunctionRule.java | 6 +-
.../query/optrule/KapAggFilterTransposeRule.java | 4 +-
.../kap/query/optrule/KapAggJoinTransposeRule.java | 4 +-
.../kap/query/optrule/KapAggProjectMergeRule.java | 4 +-
.../query/optrule/KapAggProjectTransposeRule.java | 4 +-
.../query/optrule/KapCountDistinctJoinRule.java | 4 +-
.../kap/query/optrule/KapSumCastTransposeRule.java | 4 +-
.../kylin/query/engine/QueryRoutingEngine.java | 7 +-
.../org/apache/kylin/query/util/QueryHelper.java | 15 +-
.../org/apache/kylin/query/util/QueryUtil.java | 171 -------------
.../apache/kylin/query/util/KapQueryUtilTest.java | 60 -----
.../org/apache/kylin/query/util/QueryUtilTest.java | 171 +++++++++----
.../kap/secondstorage/tdvt/TDVTHiveTest.java | 9 +-
.../service/ModelServiceWithSecondStorageTest.java | 4 +-
.../engine/spark/builder/SegmentFlatTable.scala | 35 ++-
.../kylin/engine/spark/job/FlatTableHelper.scala | 18 +-
.../job/stage/build/FlatTableAndDictBase.scala | 12 +-
.../kylin/query/pushdown/SparkSqlClient.scala | 53 ++--
.../kylin/query/runtime/plan/ResultPlan.scala | 47 ++--
.../org/apache/kylin/common/CompareSupport.scala | 4 +-
.../org/apache/kylin/common/QuerySupport.scala | 11 +-
.../scala/org/apache/kylin/common/SSSource.scala | 8 +-
.../org/apache/kylin/common/YarnSupport.scala | 1 -
58 files changed, 720 insertions(+), 831 deletions(-)
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 309ce19ced..0802fc242e 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -2864,7 +2864,7 @@ public abstract class KylinConfigBase implements Serializable {
return TimeUtil.timeStringAs(this.getOptional("kylin.query.async.result-retain-days", "7d"), TimeUnit.DAYS);
}
- public Boolean isUniqueAsyncQueryYarnQueue() {
+ public boolean isUniqueAsyncQueryYarnQueue() {
return Boolean.parseBoolean(this.getOptional("kylin.query.unique-async-query-yarn-queue-enabled", FALSE));
}
@@ -3674,7 +3674,7 @@ public abstract class KylinConfigBase implements Serializable {
return Boolean.parseBoolean(getOptional("kylin.build.segment-overlap-enabled", FALSE));
}
- public boolean getDDLEnabled(){
+ public boolean getDDLEnabled() {
return Boolean.parseBoolean(getOptional("kylin.source.ddl.enabled", FALSE));
}
}
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java
index 1202ba6b07..a8bf64c8d4 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java
@@ -326,6 +326,10 @@ public class TableDesc extends RootPersistentEntity implements Serializable, ISo
return getBackTickCaseSensitiveIdentity("");
}
+ public String getDoubleQuoteIdentity() {
+ return getDoubleQuoteCaseSensitiveIdentity("");
+ }
+
public String getBackTickTransactionalTableIdentity(String suffix) {
return getBackTickCaseSensitiveIdentity(TRANSACTIONAL_TABLE_NAME_SUFFIX.toUpperCase(Locale.ROOT) + suffix);
}
@@ -349,6 +353,13 @@ public class TableDesc extends RootPersistentEntity implements Serializable, ISo
this.getCaseSensitiveName() + suffix);
}
+ private String getDoubleQuoteCaseSensitiveIdentity(String suffix) {
+ return "null".equals(this.getCaseSensitiveDatabase())
+ ? String.format(Locale.ROOT, "\"%s\"", this.getCaseSensitiveName())
+ : String.format(Locale.ROOT, "\"%s\".\"%s\"", this.getCaseSensitiveDatabase(),
+ this.getCaseSensitiveName() + suffix);
+ }
+
public boolean isView() {
return StringUtils.containsIgnoreCase(tableType, TABLE_TYPE_VIEW);
}
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/project/NProjectManager.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/project/NProjectManager.java
index 233a7f7eb0..80a0670656 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/project/NProjectManager.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/project/NProjectManager.java
@@ -245,4 +245,10 @@ public class NProjectManager {
updater.modify(copy);
return updateProject(copy);
}
+
+ public static KylinConfig getProjectConfig(String project) {
+ NProjectManager projectManager = NProjectManager.getInstance(KylinConfig.getInstanceFromEnv());
+ ProjectInstance projectInstance = projectManager.getProject(project);
+ return projectInstance.getConfig();
+ }
}
diff --git a/src/data-loading-service/src/test/java/org/apache/kylin/rest/service/ModelServiceBuildTest.java b/src/data-loading-service/src/test/java/org/apache/kylin/rest/service/ModelServiceBuildTest.java
index 6c9ea32d8b..72df60801a 100644
--- a/src/data-loading-service/src/test/java/org/apache/kylin/rest/service/ModelServiceBuildTest.java
+++ b/src/data-loading-service/src/test/java/org/apache/kylin/rest/service/ModelServiceBuildTest.java
@@ -94,8 +94,8 @@ import org.apache.kylin.metadata.model.util.ExpandableMeasureUtil;
import org.apache.kylin.metadata.query.QueryTimesResponse;
import org.apache.kylin.metadata.realization.RealizationStatusEnum;
import org.apache.kylin.metadata.recommendation.candidate.JdbcRawRecStore;
-import org.apache.kylin.query.util.KapQueryUtil;
import org.apache.kylin.query.util.PushDownUtil;
+import org.apache.kylin.query.util.QueryUtil;
import org.apache.kylin.rest.config.initialize.ModelBrokenListener;
import org.apache.kylin.rest.exception.BadRequestException;
import org.apache.kylin.rest.request.ModelRequest;
@@ -200,7 +200,7 @@ public class ModelServiceBuildTest extends SourceTestCase {
ReflectionTestUtils.setField(modelBuildService, "userGroupService", userGroupService);
ReflectionTestUtils.setField(semanticService, "expandableMeasureUtil",
new ExpandableMeasureUtil((model, ccDesc) -> {
- String ccExpression = KapQueryUtil.massageComputedColumn(model, model.getProject(), ccDesc,
+ String ccExpression = QueryUtil.massageComputedColumn(model, model.getProject(), ccDesc,
AclPermissionUtil.prepareQueryContextACLInfo(model.getProject(),
semanticService.getCurrentUserGroups()));
ccDesc.setInnerExpression(ccExpression);
diff --git a/src/kylin-it/src/test/java/org/apache/kylin/newten/NBadQueryAndPushDownTest.java b/src/kylin-it/src/test/java/org/apache/kylin/newten/NBadQueryAndPushDownTest.java
index fc56ec0377..22072e7d93 100644
--- a/src/kylin-it/src/test/java/org/apache/kylin/newten/NBadQueryAndPushDownTest.java
+++ b/src/kylin-it/src/test/java/org/apache/kylin/newten/NBadQueryAndPushDownTest.java
@@ -36,9 +36,9 @@ import org.apache.kylin.engine.spark.NLocalWithSparkSessionTest;
import org.apache.kylin.job.impl.threadpool.NDefaultScheduler;
import org.apache.kylin.metadata.querymeta.SelectedColumnMeta;
import org.apache.kylin.metadata.realization.NoRealizationFoundException;
-import org.apache.kylin.query.util.KapQueryUtil;
import org.apache.kylin.query.util.PushDownUtil;
import org.apache.kylin.query.util.QueryParams;
+import org.apache.kylin.query.util.QueryUtil;
import org.apache.kylin.util.ExecAndComp;
import org.apache.spark.sql.AnalysisException;
import org.apache.spark.sql.SparderEnv;
@@ -253,8 +253,7 @@ public class NBadQueryAndPushDownTest extends NLocalWithSparkSessionTest {
int offset, SQLException sqlException, boolean isForced) throws Exception {
populateSSWithCSVData(KylinConfig.getInstanceFromEnv(), prjName, SparderEnv.getSparkSession());
String pushdownSql = ExecAndComp.removeDataBaseInSql(sql);
- String massagedSql = KapQueryUtil.normalMassageSql(KylinConfig.getInstanceFromEnv(), pushdownSql, limit,
- offset);
+ String massagedSql = QueryUtil.normalMassageSql(KylinConfig.getInstanceFromEnv(), pushdownSql, limit, offset);
QueryParams queryParams = new QueryParams(prjName, massagedSql, "DEFAULT", BackdoorToggles.getPrepareOnly(),
sqlException, isForced);
queryParams.setSelect(true);
diff --git a/src/kylin-it/src/test/java/org/apache/kylin/newten/SlowQueryDetectorTest.java b/src/kylin-it/src/test/java/org/apache/kylin/newten/SlowQueryDetectorTest.java
index f5fa37480c..3e478f940c 100644
--- a/src/kylin-it/src/test/java/org/apache/kylin/newten/SlowQueryDetectorTest.java
+++ b/src/kylin-it/src/test/java/org/apache/kylin/newten/SlowQueryDetectorTest.java
@@ -32,12 +32,13 @@ import org.apache.kylin.common.util.RandomUtil;
import org.apache.kylin.engine.spark.NLocalWithSparkSessionTest;
import org.apache.kylin.job.engine.JobEngineConfig;
import org.apache.kylin.job.impl.threadpool.NDefaultScheduler;
+import org.apache.kylin.metadata.project.NProjectManager;
import org.apache.kylin.query.SlowQueryDetector;
import org.apache.kylin.query.engine.QueryExec;
import org.apache.kylin.query.pushdown.SparkSqlClient;
import org.apache.kylin.query.runtime.plan.ResultPlan;
-import org.apache.kylin.query.util.KapQueryUtil;
import org.apache.kylin.query.util.QueryParams;
+import org.apache.kylin.query.util.QueryUtil;
import org.apache.kylin.util.ExecAndComp;
import org.apache.spark.sql.SparderEnv;
import org.junit.After;
@@ -116,7 +117,7 @@ public class SlowQueryDetectorTest extends NLocalWithSparkSessionTest {
Assert.assertTrue(QueryContext.current().getQueryTagInfo().isTimeout());
Assert.assertTrue(e instanceof KylinTimeoutException);
Assert.assertEquals(
- "The query exceeds the set time limit of 300s. Current step: Collecting dataset for sparder. ",
+ "The query exceeds the set time limit of 300s. Current step: Collecting dataset for sparder.",
e.getMessage());
// reset query thread's interrupt state.
Thread.interrupted();
@@ -139,7 +140,7 @@ public class SlowQueryDetectorTest extends NLocalWithSparkSessionTest {
Assert.assertTrue(QueryContext.current().getQueryTagInfo().isTimeout());
Assert.assertTrue(e instanceof KylinTimeoutException);
Assert.assertEquals(
- "The query exceeds the set time limit of 300s. Current step: Collecting dataset for push-down. ",
+ "The query exceeds the set time limit of 300s. Current step: Collecting dataset of push-down.",
e.getMessage());
// reset query thread's interrupt state.
Thread.interrupted();
@@ -156,9 +157,9 @@ public class SlowQueryDetectorTest extends NLocalWithSparkSessionTest {
long t = System.currentTimeMillis();
String sql = FileUtils
.readFileToString(new File("src/test/resources/query/sql_timeout/query03.sql"), "UTF-8").trim();
- QueryParams queryParams = new QueryParams(KapQueryUtil.getKylinConfig(getProject()), sql, getProject(), 0,
- 0, "DEFAULT", true);
- KapQueryUtil.massageSql(queryParams);
+ QueryParams queryParams = new QueryParams(NProjectManager.getProjectConfig(getProject()), sql, getProject(),
+ 0, 0, "DEFAULT", true);
+ QueryUtil.massageSql(queryParams);
String error = "TestSQLMassageTimeoutCancelJob fail, query cost:" + (System.currentTimeMillis() - t)
+ " ms, need compute:" + SparderEnv.needCompute();
logger.error(error);
diff --git a/src/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java b/src/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
index e4776616fc..c538c0dd1e 100644
--- a/src/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
+++ b/src/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java
@@ -44,9 +44,9 @@ import org.apache.kylin.common.util.Pair;
import org.apache.kylin.metadata.project.ProjectInstance;
import org.apache.kylin.metadata.querymeta.SelectedColumnMeta;
import org.apache.kylin.query.engine.QueryExec;
-import org.apache.kylin.query.util.KapQueryUtil;
import org.apache.kylin.query.util.PushDownUtil;
import org.apache.kylin.query.util.QueryParams;
+import org.apache.kylin.query.util.QueryUtil;
import org.apache.kylin.util.ExecAndComp;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -134,7 +134,7 @@ public class KylinTestBase extends NLocalFileMetadataTestCase {
public Pair<List<List<String>>, List<SelectedColumnMeta>> tryPushDownSelectQuery(String project, String sql,
String defaultSchema, SQLException sqlException, boolean isPrepare, boolean isForced) throws Exception {
- String massagedSql = KapQueryUtil.normalMassageSql(KylinConfig.getInstanceFromEnv(), sql, 0, 0);
+ String massagedSql = QueryUtil.normalMassageSql(KylinConfig.getInstanceFromEnv(), sql, 0, 0);
QueryParams queryParams = new QueryParams(project, massagedSql, defaultSchema, isPrepare, sqlException,
isForced);
queryParams.setSelect(true);
diff --git a/src/kylin-it/src/test/java/org/apache/kylin/query/rules/CalciteRuleTestBase.java b/src/kylin-it/src/test/java/org/apache/kylin/query/rules/CalciteRuleTestBase.java
index 69ef58baaa..692dccf9ca 100644
--- a/src/kylin-it/src/test/java/org/apache/kylin/query/rules/CalciteRuleTestBase.java
+++ b/src/kylin-it/src/test/java/org/apache/kylin/query/rules/CalciteRuleTestBase.java
@@ -45,8 +45,8 @@ import org.apache.kylin.common.util.Pair;
import org.apache.kylin.query.engine.QueryExec;
import org.apache.kylin.query.engine.QueryOptimizer;
import org.apache.kylin.query.util.HepUtils;
-import org.apache.kylin.query.util.KapQueryUtil;
import org.apache.kylin.query.util.QueryParams;
+import org.apache.kylin.query.util.QueryUtil;
import org.apache.kylin.util.ExecAndComp;
import org.junit.Assert;
import org.slf4j.Logger;
@@ -111,7 +111,7 @@ public class CalciteRuleTestBase extends NLocalFileMetadataTestCase {
return e.getFirst().contains(file);
}).map(e -> {
QueryParams queryParams = new QueryParams(config, e.getSecond(), project, 0, 0, "DEFAULT", false);
- String sql = KapQueryUtil.massageSql(queryParams).replaceAll(emptyLinePattern, ""); // remove empty line
+ String sql = QueryUtil.massageSql(queryParams).replaceAll(emptyLinePattern, ""); // remove empty line
return new Pair<>(FilenameUtils.getBaseName(e.getFirst()), sql);
}).collect(Collectors.toList());
Assert.assertEquals(1, queries.size());
@@ -123,7 +123,7 @@ public class CalciteRuleTestBase extends NLocalFileMetadataTestCase {
final String queryFolder = IT_SQL_KYLIN_DIR + folder;
return ExecAndComp.fetchQueries(queryFolder).stream().map(e -> {
QueryParams queryParams = new QueryParams(config, e.getSecond(), project, 0, 0, "DEFAULT", false);
- String sql = KapQueryUtil.massageSql(queryParams).replaceAll(emptyLinePattern, ""); // remove empty line
+ String sql = QueryUtil.massageSql(queryParams).replaceAll(emptyLinePattern, ""); // remove empty line
return new Pair<>(FilenameUtils.getBaseName(e.getFirst()), sql);
}).collect(Collectors.toList());
}
diff --git a/src/kylin-it/src/test/java/org/apache/kylin/util/ExecAndComp.java b/src/kylin-it/src/test/java/org/apache/kylin/util/ExecAndComp.java
index 4e0282ae78..87f4eab000 100644
--- a/src/kylin-it/src/test/java/org/apache/kylin/util/ExecAndComp.java
+++ b/src/kylin-it/src/test/java/org/apache/kylin/util/ExecAndComp.java
@@ -46,6 +46,7 @@ import org.apache.kylin.query.engine.QueryExec;
import org.apache.kylin.query.engine.data.QueryResult;
import org.apache.kylin.query.relnode.OLAPContext;
import org.apache.kylin.query.util.QueryParams;
+import org.apache.kylin.query.util.QueryUtil;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Row;
import org.apache.spark.sql.SparderEnv;
@@ -55,7 +56,6 @@ import org.apache.spark.sql.util.SparderTypeUtil;
import io.kyligence.kap.guava20.shaded.common.base.Preconditions;
import io.kyligence.kap.guava20.shaded.common.collect.Lists;
import io.kyligence.kap.guava20.shaded.common.collect.Sets;
-import org.apache.kylin.query.util.KapQueryUtil;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
@@ -174,8 +174,8 @@ public class ExecAndComp {
}
QueryParams queryParams = new QueryParams(prj, compareSql, "default", false);
- queryParams.setKylinConfig(KapQueryUtil.getKylinConfig(prj));
- String afterConvert = KapQueryUtil.massagePushDownSql(queryParams);
+ queryParams.setKylinConfig(NProjectManager.getProjectConfig(prj));
+ String afterConvert = QueryUtil.massagePushDownSql(queryParams);
// Table schema comes from csv and DATABASE.TABLE is not supported.
String sqlForSpark = removeDataBaseInSql(afterConvert);
val ds = querySparkSql(sqlForSpark);
@@ -304,9 +304,9 @@ public class ExecAndComp {
public static Dataset<Row> queryModelWithoutCompute(String prj, String sql, List<String> parameters) {
try {
SparderEnv.skipCompute();
- QueryParams queryParams = new QueryParams(KapQueryUtil.getKylinConfig(prj), sql, prj, 0, 0, "DEFAULT",
+ QueryParams queryParams = new QueryParams(NProjectManager.getProjectConfig(prj), sql, prj, 0, 0, "DEFAULT",
true);
- sql = KapQueryUtil.massageSql(queryParams);
+ sql = QueryUtil.massageSql(queryParams);
List<String> parametersNotNull = parameters == null ? new ArrayList<>() : parameters;
return queryModel(prj, sql, parametersNotNull);
} finally {
@@ -336,9 +336,9 @@ public class ExecAndComp {
}
public static QueryResult queryModelWithMassage(String prj, String sqlText, List<String> parameters) {
- QueryParams queryParams = new QueryParams(KapQueryUtil.getKylinConfig(prj), sqlText, prj, 0, 0, "DEFAULT",
+ QueryParams queryParams = new QueryParams(NProjectManager.getProjectConfig(prj), sqlText, prj, 0, 0, "DEFAULT",
true);
- sqlText = KapQueryUtil.massageSql(queryParams);
+ sqlText = QueryUtil.massageSql(queryParams);
if (sqlText == null)
throw new RuntimeException("Sorry your SQL is null...");
@@ -402,7 +402,7 @@ public class ExecAndComp {
}
public static void execAndCompareQueryList(List<String> queries, String prj, CompareLevel compareLevel,
- String joinType) {
+ String joinType) {
List<Pair<String, String>> transformed = queries.stream().map(q -> Pair.newPair("", q))
.collect(Collectors.toList());
execAndCompare(transformed, prj, compareLevel, joinType);
diff --git a/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelSemanticHelper.java b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelSemanticHelper.java
index 808b732f7a..4decdbb834 100644
--- a/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelSemanticHelper.java
+++ b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelSemanticHelper.java
@@ -93,6 +93,8 @@ import org.apache.kylin.metadata.model.util.scd2.SCD2NonEquiCondSimplification;
import org.apache.kylin.metadata.model.util.scd2.SimplifiedJoinDesc;
import org.apache.kylin.metadata.model.util.scd2.SimplifiedJoinTableDesc;
import org.apache.kylin.metadata.project.NProjectManager;
+import org.apache.kylin.metadata.recommendation.ref.OptRecManagerV2;
+import org.apache.kylin.query.util.QueryUtil;
import org.apache.kylin.rest.request.ModelRequest;
import org.apache.kylin.rest.response.BuildIndexResponse;
import org.apache.kylin.rest.response.SimplifiedMeasure;
@@ -111,8 +113,6 @@ import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
-import org.apache.kylin.metadata.recommendation.ref.OptRecManagerV2;
-import org.apache.kylin.query.util.KapQueryUtil;
import io.kyligence.kap.secondstorage.SecondStorageUpdater;
import io.kyligence.kap.secondstorage.SecondStorageUtil;
import lombok.Setter;
@@ -130,7 +130,7 @@ public class ModelSemanticHelper extends BasicService {
private static final Logger logger = LoggerFactory.getLogger(ModelSemanticHelper.class);
private final ExpandableMeasureUtil expandableMeasureUtil = new ExpandableMeasureUtil((model, ccDesc) -> {
- String ccExpression = KapQueryUtil.massageComputedColumn(model, model.getProject(), ccDesc,
+ String ccExpression = QueryUtil.massageComputedColumn(model, model.getProject(), ccDesc,
AclPermissionUtil.prepareQueryContextACLInfo(model.getProject(), getCurrentUserGroups()));
ccDesc.setInnerExpression(ccExpression);
ComputedColumnEvalUtil.evaluateExprAndType(model, ccDesc);
diff --git a/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelService.java b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelService.java
index e4af48b957..ab859e89b0 100644
--- a/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelService.java
+++ b/src/modeling-service/src/main/java/org/apache/kylin/rest/service/ModelService.java
@@ -198,9 +198,9 @@ import org.apache.kylin.metadata.project.NProjectManager;
import org.apache.kylin.metadata.project.ProjectInstance;
import org.apache.kylin.metadata.realization.RealizationStatusEnum;
import org.apache.kylin.metadata.streaming.KafkaConfig;
-import org.apache.kylin.query.util.KapQueryUtil;
import org.apache.kylin.query.util.PushDownUtil;
import org.apache.kylin.query.util.QueryParams;
+import org.apache.kylin.query.util.QueryUtil;
import org.apache.kylin.rest.aspect.Transaction;
import org.apache.kylin.rest.constant.ModelAttributeEnum;
import org.apache.kylin.rest.constant.ModelStatusToDisplayEnum;
@@ -1738,7 +1738,7 @@ public class ModelService extends AbstractModelService implements TableModelSupp
queryParams.setKylinConfig(projectInstance.getConfig());
queryParams.setAclInfo(
AclPermissionUtil.prepareQueryContextACLInfo(dataModel.getProject(), getCurrentUserGroups()));
- String pushdownSql = KapQueryUtil.massagePushDownSql(queryParams);
+ String pushdownSql = QueryUtil.massagePushDownSql(queryParams);
ss.sql(pushdownSql);
}
} catch (Exception e) {
@@ -2618,7 +2618,7 @@ public class ModelService extends AbstractModelService implements TableModelSupp
model.init(getConfig(), project, getManager(NDataModelManager.class, project).getCCRelatedModels(model));
model.getComputedColumnDescs().forEach(cc -> {
- String innerExp = KapQueryUtil.massageComputedColumn(model, project, cc, null);
+ String innerExp = QueryUtil.massageComputedColumn(model, project, cc, null);
cc.setInnerExpression(innerExp);
});
@@ -2645,7 +2645,7 @@ public class ModelService extends AbstractModelService implements TableModelSupp
MsgPicker.getMsg().getccOnAntiFlattenLookup(), antiFlattenLookup));
}
ComputedColumnDesc.simpleParserCheck(cc.getExpression(), model.getAliasMap().keySet());
- String innerExpression = KapQueryUtil.massageComputedColumn(model, project, cc,
+ String innerExpression = QueryUtil.massageComputedColumn(model, project, cc,
AclPermissionUtil.prepareQueryContextACLInfo(project, getCurrentUserGroups()));
cc.setInnerExpression(innerExpression);
@@ -2747,7 +2747,7 @@ public class ModelService extends AbstractModelService implements TableModelSupp
// Update CC expression from query transformers
for (ComputedColumnDesc ccDesc : model.getComputedColumnDescs()) {
- String ccExpression = KapQueryUtil.massageComputedColumn(model, project, ccDesc,
+ String ccExpression = QueryUtil.massageComputedColumn(model, project, ccDesc,
AclPermissionUtil.prepareQueryContextACLInfo(project, getCurrentUserGroups()));
ccDesc.setInnerExpression(ccExpression);
TblColRef tblColRef = model.findColumn(ccDesc.getTableAlias(), ccDesc.getColumnName());
@@ -3598,8 +3598,7 @@ public class ModelService extends AbstractModelService implements TableModelSupp
return;
}
- String massagedFilterCond = KapQueryUtil.massageExpression(model, model.getProject(),
- model.getFilterCondition(),
+ String massagedFilterCond = QueryUtil.massageExpression(model, model.getProject(), model.getFilterCondition(),
AclPermissionUtil.prepareQueryContextACLInfo(model.getProject(), getCurrentUserGroups()), false);
String filterConditionWithTableName = addTableNameIfNotExist(massagedFilterCond, model);
@@ -4170,7 +4169,7 @@ public class ModelService extends AbstractModelService implements TableModelSupp
for (ComputedColumnDesc cc : model.getComputedColumnDescs()) {
String innerExp = cc.getInnerExpression();
if (cc.getExpression().equalsIgnoreCase(innerExp)) {
- innerExp = KapQueryUtil.massageComputedColumn(model, project, cc, null);
+ innerExp = QueryUtil.massageComputedColumn(model, project, cc, null);
}
cc.setInnerExpression(innerExp);
}
diff --git a/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelServiceTest.java b/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelServiceTest.java
index 8d3a4462b2..e55b82cdfe 100644
--- a/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelServiceTest.java
+++ b/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelServiceTest.java
@@ -157,7 +157,7 @@ import org.apache.kylin.metadata.query.QueryTimesResponse;
import org.apache.kylin.metadata.realization.RealizationStatusEnum;
import org.apache.kylin.metadata.recommendation.candidate.JdbcRawRecStore;
import org.apache.kylin.metadata.user.ManagedUser;
-import org.apache.kylin.query.util.KapQueryUtil;
+import org.apache.kylin.query.util.QueryUtil;
import org.apache.kylin.rest.config.initialize.ModelBrokenListener;
import org.apache.kylin.rest.constant.Constant;
import org.apache.kylin.rest.constant.ModelStatusToDisplayEnum;
@@ -285,7 +285,7 @@ public class ModelServiceTest extends SourceTestCase {
ReflectionTestUtils.setField(semanticService, "userGroupService", userGroupService);
ReflectionTestUtils.setField(semanticService, "expandableMeasureUtil",
new ExpandableMeasureUtil((model, ccDesc) -> {
- String ccExpression = KapQueryUtil.massageComputedColumn(model, model.getProject(), ccDesc,
+ String ccExpression = QueryUtil.massageComputedColumn(model, model.getProject(), ccDesc,
AclPermissionUtil.prepareQueryContextACLInfo(model.getProject(),
semanticService.getCurrentUserGroups()));
ccDesc.setInnerExpression(ccExpression);
diff --git a/src/query-common/src/main/java/org/apache/kylin/query/IQueryTransformer.java b/src/query-common/src/main/java/org/apache/kylin/query/IQueryTransformer.java
new file mode 100644
index 0000000000..f7465762dc
--- /dev/null
+++ b/src/query-common/src/main/java/org/apache/kylin/query/IQueryTransformer.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.query;
+
+public interface IQueryTransformer {
+ String transform(String sql, String project, String defaultSchema);
+}
diff --git a/src/query-common/src/main/java/org/apache/kylin/query/SlowQueryDetector.java b/src/query-common/src/main/java/org/apache/kylin/query/SlowQueryDetector.java
index 7b871b2028..3810c2d09f 100644
--- a/src/query-common/src/main/java/org/apache/kylin/query/SlowQueryDetector.java
+++ b/src/query-common/src/main/java/org/apache/kylin/query/SlowQueryDetector.java
@@ -42,7 +42,7 @@ public class SlowQueryDetector extends Thread {
private static final ConcurrentMap<String, CanceledSlowQueryStatus> canceledSlowQueriesStatus = Maps
.newConcurrentMap();
private final int detectionIntervalMs;
- private int queryTimeoutMs;
+ private final int queryTimeoutMs;
public SlowQueryDetector() {
super("SlowQueryDetector");
diff --git a/src/query-common/src/main/java/org/apache/kylin/query/security/HackSelectStarWithColumnACL.java b/src/query-common/src/main/java/org/apache/kylin/query/security/HackSelectStarWithColumnACL.java
index 1b7c160e69..5fe9a10fa7 100644
--- a/src/query-common/src/main/java/org/apache/kylin/query/security/HackSelectStarWithColumnACL.java
+++ b/src/query-common/src/main/java/org/apache/kylin/query/security/HackSelectStarWithColumnACL.java
@@ -37,21 +37,21 @@ import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.QueryContext;
import org.apache.kylin.common.exception.KylinRuntimeException;
import org.apache.kylin.common.util.Pair;
+import org.apache.kylin.metadata.acl.AclTCR;
+import org.apache.kylin.metadata.acl.AclTCRManager;
import org.apache.kylin.metadata.model.ColumnDesc;
+import org.apache.kylin.metadata.model.NTableMetadataManager;
import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.metadata.model.tool.CalciteParser;
+import org.apache.kylin.query.IQueryTransformer;
+import org.apache.kylin.query.exception.NoAuthorizedColsError;
import org.apache.kylin.rest.constant.Constant;
import org.apache.kylin.source.adhocquery.IPushDownConverter;
-import org.apache.kylin.metadata.acl.AclTCR;
-import org.apache.kylin.metadata.acl.AclTCRManager;
-import org.apache.kylin.metadata.model.NTableMetadataManager;
-import org.apache.kylin.query.exception.NoAuthorizedColsError;
-import org.apache.kylin.query.util.KapQueryUtil;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
-public class HackSelectStarWithColumnACL implements KapQueryUtil.IQueryTransformer, IPushDownConverter {
+public class HackSelectStarWithColumnACL implements IQueryTransformer, IPushDownConverter {
private static final String SELECT_STAR = "*";
diff --git a/src/query-common/src/main/java/org/apache/kylin/query/security/RowFilter.java b/src/query-common/src/main/java/org/apache/kylin/query/security/RowFilter.java
index 98005174df..bb5525a3a7 100644
--- a/src/query-common/src/main/java/org/apache/kylin/query/security/RowFilter.java
+++ b/src/query-common/src/main/java/org/apache/kylin/query/security/RowFilter.java
@@ -44,17 +44,17 @@ import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.QueryContext;
import org.apache.kylin.common.exception.KylinRuntimeException;
import org.apache.kylin.common.util.Pair;
+import org.apache.kylin.metadata.acl.AclTCRManager;
import org.apache.kylin.metadata.model.tool.CalciteParser;
+import org.apache.kylin.query.IQueryTransformer;
import org.apache.kylin.rest.constant.Constant;
import org.apache.kylin.source.adhocquery.IPushDownConverter;
-import org.apache.kylin.metadata.acl.AclTCRManager;
-import org.apache.kylin.query.util.KapQueryUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.common.base.Preconditions;
-public class RowFilter implements KapQueryUtil.IQueryTransformer, IPushDownConverter {
+public class RowFilter implements IQueryTransformer, IPushDownConverter {
private static final Logger logger = LoggerFactory.getLogger(RowFilter.class);
static boolean needEscape(String sql, String defaultSchema, Map<String, String> cond) {
diff --git a/src/query-common/src/main/java/org/apache/kylin/query/util/ConvertToComputedColumn.java b/src/query-common/src/main/java/org/apache/kylin/query/util/ConvertToComputedColumn.java
index ad40e547e3..1958ccd59c 100644
--- a/src/query-common/src/main/java/org/apache/kylin/query/util/ConvertToComputedColumn.java
+++ b/src/query-common/src/main/java/org/apache/kylin/query/util/ConvertToComputedColumn.java
@@ -52,11 +52,12 @@ import org.apache.kylin.common.KapConfig;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.Pair;
import org.apache.kylin.common.util.ThreadUtil;
-import org.apache.kylin.metadata.model.tool.CalciteParser;
import org.apache.kylin.metadata.cube.model.NDataflowManager;
import org.apache.kylin.metadata.model.ComputedColumnDesc;
import org.apache.kylin.metadata.model.NDataModel;
import org.apache.kylin.metadata.model.alias.ExpressionComparator;
+import org.apache.kylin.metadata.model.tool.CalciteParser;
+import org.apache.kylin.query.IQueryTransformer;
import com.google.common.base.Function;
import com.google.common.collect.ImmutableList;
@@ -71,7 +72,7 @@ import lombok.val;
import lombok.extern.slf4j.Slf4j;
@Slf4j
-public class ConvertToComputedColumn implements KapQueryUtil.IQueryTransformer {
+public class ConvertToComputedColumn implements IQueryTransformer {
private static final String CONVERT_TO_CC_ERROR_MSG = "Something unexpected while ConvertToComputedColumn transforming the query, return original query.";
@@ -379,9 +380,9 @@ public class ConvertToComputedColumn implements KapQueryUtil.IQueryTransformer {
}
try {
SqlNode inputNodes = CalciteParser.parse(inputSql);
- int cntNodesBefore = getInputTreeNodes((SqlCall) inputNodes).size();
+ int cntNodesBefore = getInputTreeNodes(inputNodes).size();
SqlNode resultNodes = CalciteParser.parse(result);
- int cntNodesAfter = getInputTreeNodes((SqlCall) resultNodes).size();
+ int cntNodesAfter = getInputTreeNodes(resultNodes).size();
return Pair.newPair(result, cntNodesBefore - cntNodesAfter);
} catch (SqlParseException e) {
log.debug("Convert to computedColumn Fail, parse result sql fail: {}", result, e);
@@ -569,7 +570,7 @@ public class ConvertToComputedColumn implements KapQueryUtil.IQueryTransformer {
}
public void replace(String sql, boolean replaceCcName) {
- SqlSelect sqlSelect = KapQueryUtil.extractSqlSelect(selectOrOrderby);
+ SqlSelect sqlSelect = QueryUtil.extractSqlSelect(selectOrOrderby);
if (sqlSelect == null) {
return;
}
@@ -588,14 +589,11 @@ public class ConvertToComputedColumn implements KapQueryUtil.IQueryTransformer {
if (replaceCcName && !sql.equals(ret.getFirst())) {
choiceForCurrentSubquery = ret;
- } else {
- if (ret.getSecond() == 0) {
- continue;
- }
- if (choiceForCurrentSubquery == null || ret.getSecond() > choiceForCurrentSubquery.getSecond()) {
- choiceForCurrentSubquery = ret;
- recursionCompleted = false;
- }
+ } else if (ret.getSecond() != 0 //
+ && (choiceForCurrentSubquery == null
+ || ret.getSecond() > choiceForCurrentSubquery.getSecond())) {
+ choiceForCurrentSubquery = ret;
+ recursionCompleted = false;
}
}
}
diff --git a/src/query-common/src/main/java/org/apache/kylin/query/util/DateNumberFilterTransformer.java b/src/query-common/src/main/java/org/apache/kylin/query/util/DateNumberFilterTransformer.java
index cdbb1fd336..3c75152419 100644
--- a/src/query-common/src/main/java/org/apache/kylin/query/util/DateNumberFilterTransformer.java
+++ b/src/query-common/src/main/java/org/apache/kylin/query/util/DateNumberFilterTransformer.java
@@ -34,10 +34,11 @@ import org.apache.calcite.sql.parser.SqlParseException;
import org.apache.calcite.util.Litmus;
import org.apache.kylin.common.util.Pair;
import org.apache.kylin.metadata.model.tool.CalciteParser;
+import org.apache.kylin.query.IQueryTransformer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-public class DateNumberFilterTransformer implements KapQueryUtil.IQueryTransformer {
+public class DateNumberFilterTransformer implements IQueryTransformer {
private static final Logger logger = LoggerFactory.getLogger(DateNumberFilterTransformer.class);
@@ -85,8 +86,8 @@ public class DateNumberFilterTransformer implements KapQueryUtil.IQueryTransform
static class SqlTimeFilterMatcher extends AbstractSqlVisitor {
private final List<Pair<String, Pair<Integer, Integer>>> timeFilterPositions = new ArrayList<>();
- private static final List<String> SUPPORT_FUN = Arrays.asList("=", "IN", "NOT IN", "BETWEEN", "NOT BETWEEN", "<", ">", "<=",
- ">=", "!=", "<>");
+ private static final List<String> SUPPORT_FUN = Arrays.asList("=", "IN", "NOT IN", "BETWEEN", "NOT BETWEEN",
+ "<", ">", "<=", ">=", "!=", "<>");
private static final List<String> YEAR_FUN = Arrays.asList("YEAR", "{fn YEAR}");
private static final List<String> MONTH_FUN = Arrays.asList("MONTH", "{fn MONTH}");
private static final List<String> DAY_FUN = Arrays.asList("DAYOFMONTH", "{fn DAYOFMONTH}");
@@ -421,8 +422,8 @@ public class DateNumberFilterTransformer implements KapQueryUtil.IQueryTransform
}
} else if (isYearMultiplicationExpression(multiplier, tmpExpression)
&& (colName == null || colName.equalsDeep(tmpExpression.operand(0), Litmus.IGNORE))) {
- colName = tmpExpression.operand(0);
- yearExpression = tmpExpression;
+ colName = tmpExpression.operand(0);
+ yearExpression = tmpExpression;
}
}
}
diff --git a/src/query-common/src/main/java/org/apache/kylin/query/util/DefaultQueryTransformer.java b/src/query-common/src/main/java/org/apache/kylin/query/util/DefaultQueryTransformer.java
index d5c91c43d3..18ec915c90 100644
--- a/src/query-common/src/main/java/org/apache/kylin/query/util/DefaultQueryTransformer.java
+++ b/src/query-common/src/main/java/org/apache/kylin/query/util/DefaultQueryTransformer.java
@@ -22,11 +22,12 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.query.IQueryTransformer;
/**
* DefaultQueryTransformer only used for query from IndexPlan.
*/
-public class DefaultQueryTransformer implements KapQueryUtil.IQueryTransformer {
+public class DefaultQueryTransformer implements IQueryTransformer {
private static final String S0 = "\\s*";
private static final String SM = "\\s+";
diff --git a/src/query-common/src/main/java/org/apache/kylin/query/util/EscapeTransformer.java b/src/query-common/src/main/java/org/apache/kylin/query/util/EscapeTransformer.java
index c1d77ce5fb..718939daf5 100644
--- a/src/query-common/src/main/java/org/apache/kylin/query/util/EscapeTransformer.java
+++ b/src/query-common/src/main/java/org/apache/kylin/query/util/EscapeTransformer.java
@@ -19,10 +19,11 @@
package org.apache.kylin.query.util;
import org.apache.kylin.common.KapConfig;
+import org.apache.kylin.query.IQueryTransformer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-public class EscapeTransformer implements KapQueryUtil.IQueryTransformer {
+public class EscapeTransformer implements IQueryTransformer {
private static final Logger logger = LoggerFactory.getLogger(EscapeTransformer.class);
@@ -51,7 +52,8 @@ public class EscapeTransformer implements KapQueryUtil.IQueryTransformer {
logger.debug("EscapeParser done parsing");
return result;
} catch (Throwable ex) {
- logger.error("Something unexpected while EscapeTransformer transforming the query, return original query", ex);
+ logger.error("Something unexpected while EscapeTransformer transforming the query, return original query",
+ ex);
logger.error(sql);
return sql;
}
diff --git a/src/query-common/src/main/java/org/apache/kylin/query/util/KeywordDefaultDirtyHack.java b/src/query-common/src/main/java/org/apache/kylin/query/util/KeywordDefaultDirtyHack.java
index 662be9cd41..367c1bf858 100644
--- a/src/query-common/src/main/java/org/apache/kylin/query/util/KeywordDefaultDirtyHack.java
+++ b/src/query-common/src/main/java/org/apache/kylin/query/util/KeywordDefaultDirtyHack.java
@@ -19,9 +19,10 @@
package org.apache.kylin.query.util;
import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.query.IQueryTransformer;
import org.apache.kylin.source.adhocquery.IPushDownConverter;
-public class KeywordDefaultDirtyHack implements KapQueryUtil.IQueryTransformer, IPushDownConverter {
+public class KeywordDefaultDirtyHack implements IQueryTransformer, IPushDownConverter {
public static String transform(String sql) {
// KYLIN-2108, DEFAULT is hive default database, but a sql keyword too,
diff --git a/src/query-common/src/main/java/org/apache/kylin/query/util/PowerBIConverter.java b/src/query-common/src/main/java/org/apache/kylin/query/util/PowerBIConverter.java
index 5cc42b25e7..d2ad2f63a6 100644
--- a/src/query-common/src/main/java/org/apache/kylin/query/util/PowerBIConverter.java
+++ b/src/query-common/src/main/java/org/apache/kylin/query/util/PowerBIConverter.java
@@ -20,9 +20,10 @@ package org.apache.kylin.query.util;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
+import org.apache.kylin.query.IQueryTransformer;
import org.apache.kylin.source.adhocquery.IPushDownConverter;
-public class PowerBIConverter implements KapQueryUtil.IQueryTransformer, IPushDownConverter {
+public class PowerBIConverter implements IQueryTransformer, IPushDownConverter {
private static final String S0 = "\\s*";
private static final String SM = "\\s+";
@@ -40,8 +41,7 @@ public class PowerBIConverter implements KapQueryUtil.IQueryTransformer, IPushDo
if (!m.find())
break;
- sql = sql.substring(0, m.start()) + " SUM(" + m.group(1).trim() + ")"
- + sql.substring(m.end(), sql.length());
+ sql = sql.substring(0, m.start()) + " SUM(" + m.group(1).trim() + ")" + sql.substring(m.end());
}
return sql;
}
diff --git a/src/query-common/src/main/java/org/apache/kylin/query/util/PushDownUtil.java b/src/query-common/src/main/java/org/apache/kylin/query/util/PushDownUtil.java
index 11a0b7a850..9fad04509b 100644
--- a/src/query-common/src/main/java/org/apache/kylin/query/util/PushDownUtil.java
+++ b/src/query-common/src/main/java/org/apache/kylin/query/util/PushDownUtil.java
@@ -52,13 +52,13 @@ import org.apache.kylin.metadata.model.ISourceAware;
import org.apache.kylin.metadata.model.NTableMetadataManager;
import org.apache.kylin.metadata.model.SegmentRange;
import org.apache.kylin.metadata.model.TableDesc;
+import org.apache.kylin.metadata.project.NProjectManager;
import org.apache.kylin.metadata.querymeta.SelectedColumnMeta;
import org.apache.kylin.metadata.realization.RoutingIndicatorException;
+import org.apache.kylin.query.exception.NoAuthorizedColsError;
import org.apache.kylin.query.security.AccessDeniedException;
import org.apache.kylin.source.adhocquery.IPushDownRunner;
import org.apache.kylin.source.adhocquery.PushdownResult;
-import org.apache.kylin.metadata.project.NProjectManager;
-import org.apache.kylin.query.exception.NoAuthorizedColsError;
import org.codehaus.commons.compiler.CompileException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -125,7 +125,7 @@ public class PushDownUtil {
queryParams.setKylinConfig(kylinConfig);
queryParams.setSql(sql);
try {
- sql = KapQueryUtil.massagePushDownSql(queryParams);
+ sql = QueryUtil.massagePushDownSql(queryParams);
} catch (NoAuthorizedColsError e) {
// on no authorized cols found, return empty result
return PushdownResult.emptyResult();
diff --git a/src/query-common/src/main/java/org/apache/kylin/query/util/QueryAliasMatcher.java b/src/query-common/src/main/java/org/apache/kylin/query/util/QueryAliasMatcher.java
index 8b059ebd41..6f370d4d55 100644
--- a/src/query-common/src/main/java/org/apache/kylin/query/util/QueryAliasMatcher.java
+++ b/src/query-common/src/main/java/org/apache/kylin/query/util/QueryAliasMatcher.java
@@ -22,8 +22,7 @@ import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
-
-import javax.annotation.Nullable;
+import java.util.stream.Collectors;
import org.apache.calcite.schema.Table;
import org.apache.calcite.sql.JoinConditionType;
@@ -41,29 +40,27 @@ import org.apache.calcite.sql.SqlOrderBy;
import org.apache.calcite.sql.SqlSelect;
import org.apache.calcite.sql.dialect.CalciteSqlDialect;
import org.apache.calcite.sql.util.SqlBasicVisitor;
+import org.apache.commons.collections.MapUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.KylinConfigExt;
import org.apache.kylin.common.util.Pair;
import org.apache.kylin.common.util.RandomUtil;
+import org.apache.kylin.metadata.cube.model.NDataflowManager;
import org.apache.kylin.metadata.model.ColumnDesc;
import org.apache.kylin.metadata.model.JoinDesc;
import org.apache.kylin.metadata.model.JoinsGraph;
-import org.apache.kylin.metadata.model.TableRef;
-import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.metadata.model.tool.CalciteParser;
-import org.apache.kylin.query.relnode.ColumnRowType;
-import org.apache.kylin.query.schema.OLAPTable;
-import org.apache.kylin.metadata.cube.model.NDataflowManager;
import org.apache.kylin.metadata.model.NDataModel;
import org.apache.kylin.metadata.model.NTableMetadataManager;
+import org.apache.kylin.metadata.model.TableRef;
+import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.model.alias.ExpressionComparator;
+import org.apache.kylin.metadata.model.tool.CalciteParser;
import org.apache.kylin.metadata.project.NProjectManager;
+import org.apache.kylin.query.relnode.ColumnRowType;
import org.apache.kylin.query.schema.KapOLAPSchema;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import org.apache.kylin.query.schema.OLAPTable;
-import com.google.common.base.Function;
import com.google.common.base.Preconditions;
import com.google.common.collect.BiMap;
import com.google.common.collect.HashBiMap;
@@ -76,13 +73,13 @@ import com.google.common.collect.Maps;
// Not designed to reuse, re-new per query
public class QueryAliasMatcher {
static final ColumnRowType MODEL_VIEW_COLUMN_ROW_TYPE = new ColumnRowType(new ArrayList<>());
- private static final Logger logger = LoggerFactory.getLogger(QueryAliasMatcher.class);
private static final ColumnRowType SUBQUERY_TAG = new ColumnRowType(null);
private static final String[] COLUMN_ARRAY_MARKER = new String[0];
private final String project;
private final String defaultSchema;
private final Map<String, KapOLAPSchema> schemaMap = Maps.newHashMap();
private final Map<String, Map<String, OLAPTable>> schemaTables = Maps.newHashMap();
+
public QueryAliasMatcher(String project, String defaultSchema) {
this.project = project;
this.defaultSchema = defaultSchema;
@@ -99,14 +96,14 @@ public class QueryAliasMatcher {
String tableAlias = namesOfIdentifier.get(1);
String colName = namesOfIdentifier.get(2);
ColumnRowType columnRowType = alias2CRT.get(tableAlias);
- Preconditions.checkState(columnRowType != null, "Alias " + tableAlias + " is not defined");
+ Preconditions.checkState(columnRowType != null, "Alias {} is not defined", tableAlias);
return columnRowType == QueryAliasMatcher.SUBQUERY_TAG ? null : columnRowType.getColumnByName(colName);
} else if (namesOfIdentifier.size() == 2) {
// tableAlias.colName
String tableAlias = namesOfIdentifier.get(0);
String colName = namesOfIdentifier.get(1);
ColumnRowType columnRowType = alias2CRT.get(tableAlias);
- Preconditions.checkState(columnRowType != null, "Alias " + tableAlias + " is not defined");
+ Preconditions.checkState(columnRowType != null, "Alias {} is not defined", tableAlias);
return columnRowType == QueryAliasMatcher.SUBQUERY_TAG ? null : columnRowType.getColumnByName(colName);
} else if (namesOfIdentifier.size() == 1) {
// only colName
@@ -138,8 +135,8 @@ public class QueryAliasMatcher {
/**
* match `sqlSelect` with the model in terms of join relations
- * @param model
- * @param sqlSelect
+ * @param model model
+ * @param sqlSelect select SqlNode
* @return QueryAliasMatchInfo with
* 1. the map(table alias in sql -> column row type)
* 2. match result map(table alias in sql -> table alias in model)
@@ -149,7 +146,7 @@ public class QueryAliasMatcher {
return null;
}
- SqlSelect subQuery = getSubquery(sqlSelect.getFrom());
+ SqlSelect subQuery = getSubQuery(sqlSelect.getFrom());
boolean reUseSubqeury = false;
// find subquery with permutation only projection
@@ -213,7 +210,7 @@ public class QueryAliasMatcher {
// try match the subquery with model
Map<String, String> matches = joinsGraph.matchAlias(model.getJoinsGraph(), projectConfig);
- if (matches == null || matches.isEmpty()) {
+ if (MapUtils.isEmpty(matches)) {
return null;
}
BiMap<String, String> aliasMapping = HashBiMap.create();
@@ -222,13 +219,13 @@ public class QueryAliasMatcher {
return new QueryAliasMatchInfo(aliasMapping, queryAlias);
}
- private SqlSelect getSubquery(SqlNode sqlNode) {
+ private SqlSelect getSubQuery(SqlNode sqlNode) {
if (sqlNode instanceof SqlSelect) {
return (SqlSelect) sqlNode;
} else if (SqlKind.UNION == sqlNode.getKind()) {
return (SqlSelect) ((SqlBasicCall) sqlNode).getOperandList().get(0);
} else if (SqlKind.AS == sqlNode.getKind()) {
- return getSubquery(((SqlBasicCall) sqlNode).getOperandList().get(0));
+ return getSubQuery(((SqlBasicCall) sqlNode).getOperandList().get(0));
}
return null;
@@ -270,9 +267,9 @@ public class QueryAliasMatcher {
//capture all the join within a SqlSelect's from clause, won't go into any subquery
private class SqlJoinCapturer extends SqlBasicVisitor<SqlNode> {
- private List<JoinDesc> joinDescs;
- private LinkedHashMap<String, ColumnRowType> alias2CRT = Maps.newLinkedHashMap(); // aliasInQuery => ColumnRowType representing the alias table
- private String modelName;
+ private final List<JoinDesc> joinDescs;
+ private final LinkedHashMap<String, ColumnRowType> alias2CRT = Maps.newLinkedHashMap(); // aliasInQuery => ColumnRowType representing the alias table
+ private final String modelName;
private boolean foundJoinOnCC = false;
@@ -458,12 +455,12 @@ public class QueryAliasMatcher {
}
}
- private class JoinConditionCapturer extends SqlBasicVisitor<SqlNode> {
+ private static class JoinConditionCapturer extends SqlBasicVisitor<SqlNode> {
private final LinkedHashMap<String, ColumnRowType> alias2CRT;
private final String joinType;
- private List<TblColRef> pks = Lists.newArrayList();
- private List<TblColRef> fks = Lists.newArrayList();
+ private final List<TblColRef> pks = Lists.newArrayList();
+ private final List<TblColRef> fks = Lists.newArrayList();
private boolean foundCC = false;
private boolean foundNonEqualJoin = false;
@@ -474,27 +471,17 @@ public class QueryAliasMatcher {
}
public JoinDesc getJoinDescs() {
- List<String> pkNames = Lists.transform(pks, new Function<TblColRef, String>() {
- @Nullable
- @Override
- public String apply(@Nullable TblColRef input) {
- return input == null ? null : input.getName();
- }
- });
- List<String> fkNames = Lists.transform(fks, new Function<TblColRef, String>() {
- @Nullable
- @Override
- public String apply(@Nullable TblColRef input) {
- return input == null ? null : input.getName();
- }
- });
+ List<String> pkNames = pks.stream().map(input -> input == null ? null : input.getName())
+ .collect(Collectors.toList());
+ List<String> fkNames = fks.stream().map(input -> input == null ? null : input.getName())
+ .collect(Collectors.toList());
JoinDesc join = new JoinDesc();
join.setType(joinType);
join.setForeignKey(fkNames.toArray(COLUMN_ARRAY_MARKER));
- join.setForeignKeyColumns(fks.toArray(new TblColRef[fks.size()]));
+ join.setForeignKeyColumns(fks.toArray(new TblColRef[0]));
join.setPrimaryKey(pkNames.toArray(COLUMN_ARRAY_MARKER));
- join.setPrimaryKeyColumns(pks.toArray(new TblColRef[pks.size()]));
+ join.setPrimaryKeyColumns(pks.toArray(new TblColRef[0]));
join.sortByFK();
return join;
}
diff --git a/src/query-common/src/main/java/org/apache/kylin/query/util/KapQueryUtil.java b/src/query-common/src/main/java/org/apache/kylin/query/util/QueryUtil.java
similarity index 68%
rename from src/query-common/src/main/java/org/apache/kylin/query/util/KapQueryUtil.java
rename to src/query-common/src/main/java/org/apache/kylin/query/util/QueryUtil.java
index e3226c8979..ed30dc216c 100644
--- a/src/query-common/src/main/java/org/apache/kylin/query/util/KapQueryUtil.java
+++ b/src/query-common/src/main/java/org/apache/kylin/query/util/QueryUtil.java
@@ -18,11 +18,18 @@
package org.apache.kylin.query.util;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-import com.google.common.collect.Sets;
-import io.kyligence.kap.guava20.shaded.common.collect.ImmutableSet;
-import lombok.extern.slf4j.Slf4j;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
import org.apache.calcite.plan.hep.HepRelVertex;
import org.apache.calcite.plan.volcano.RelSubset;
import org.apache.calcite.rel.RelNode;
@@ -41,7 +48,7 @@ import org.apache.calcite.sql.SqlKind;
import org.apache.calcite.sql.SqlOrderBy;
import org.apache.calcite.sql.SqlSelect;
import org.apache.calcite.util.Util;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KapConfig;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.QueryContext;
@@ -55,41 +62,137 @@ import org.apache.kylin.metadata.model.NDataModel;
import org.apache.kylin.metadata.model.TableRef;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.project.NProjectManager;
-import org.apache.kylin.metadata.project.ProjectInstance;
import org.apache.kylin.metadata.query.BigQueryThresholdUpdater;
+import org.apache.kylin.query.IQueryTransformer;
import org.apache.kylin.query.SlowQueryDetector;
import org.apache.kylin.query.exception.UserStopQueryException;
import org.apache.kylin.query.relnode.KapJoinRel;
+import org.apache.kylin.query.security.AccessDeniedException;
import org.apache.kylin.source.adhocquery.IPushDownConverter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Objects;
-import java.util.Set;
-import java.util.regex.Pattern;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
+
+public class QueryUtil {
-@Slf4j
-public class KapQueryUtil {
+ private QueryUtil() {
+ }
+
+ private static final Logger log = LoggerFactory.getLogger("query");
public static final String DEFAULT_SCHEMA = "DEFAULT";
public static final ImmutableSet<String> REMOVED_TRANSFORMERS = ImmutableSet.of("ReplaceStringWithVarchar");
-
+ private static final Pattern SELECT_PATTERN = Pattern.compile("^select", Pattern.CASE_INSENSITIVE);
+ private static final Pattern SELECT_STAR_PTN = Pattern.compile("^select\\s+\\*\\p{all}*", Pattern.CASE_INSENSITIVE);
+ private static final Pattern LIMIT_PATTERN = Pattern.compile("(limit\\s+\\d+)$", Pattern.CASE_INSENSITIVE);
+ private static final String SELECT = "select";
+ private static final String COLON = ":";
+ private static final String SEMI_COLON = ";";
public static final String JDBC = "jdbc";
public static List<IQueryTransformer> queryTransformers = Collections.emptyList();
public static List<IPushDownConverter> pushDownConverters = Collections.emptyList();
+ public static boolean isSelectStatement(String sql) {
+ String sql1 = sql.toLowerCase(Locale.ROOT);
+ sql1 = removeCommentInSql(sql1);
+ sql1 = sql1.trim();
+ while (sql1.startsWith("(")) {
+ sql1 = sql1.substring(1).trim();
+ }
+
+ return sql1.startsWith(SELECT) || (sql1.startsWith("with") && sql1.contains(SELECT))
+ || (sql1.startsWith("explain") && sql1.contains(SELECT));
+ }
+
+ public static String removeCommentInSql(String sql) {
+ // match two patterns, one is "-- comment", the other is "/* comment */"
+ try {
+ return new RawSqlParser(sql).parse().getStatementString();
+ } catch (Exception ex) {
+ log.error("Something unexpected while removing comments in the query, return original query", ex);
+ return sql;
+ }
+ }
+
+ public static String makeErrorMsgUserFriendly(Throwable e) {
+ String msg = e.getMessage();
+
+ // pick ParseException error message if possible
+ Throwable cause = e;
+ boolean needBreak = false;
+ while (cause != null) {
+ String className = cause.getClass().getName();
+ if (className.contains("ParseException") || className.contains("NoSuchTableException")
+ || className.contains("NoSuchDatabaseException") || cause instanceof AccessDeniedException) {
+ msg = cause.getMessage();
+ needBreak = true;
+ } else if (className.contains("ArithmeticException")) {
+ msg = "ArithmeticException: " + cause.getMessage();
+ needBreak = true;
+ } else if (className.contains("NoStreamingRealizationFoundException")) {
+ msg = "NoStreamingRealizationFoundException: " + cause.getMessage();
+ needBreak = true;
+ }
+ if (needBreak) {
+ break;
+ }
+ cause = cause.getCause();
+ }
+
+ return makeErrorMsgUserFriendly(msg);
+ }
+
+ public static String makeErrorMsgUserFriendly(String errorMsg) {
+ if (StringUtils.isBlank(errorMsg)) {
+ return errorMsg;
+ }
+ errorMsg = errorMsg.trim();
+ String[] split = errorMsg.split(COLON);
+ if (split.length == 3) {
+ String prefix = "Error";
+ if (StringUtils.startsWithIgnoreCase(split[0], prefix)) {
+ split[0] = split[0].substring(prefix.length()).trim();
+ }
+ prefix = "while executing SQL";
+ if (StringUtils.startsWith(split[0], prefix)) {
+ split[0] = split[0].substring(0, prefix.length()) + COLON + split[0].substring(prefix.length());
+ }
+ return split[1].trim() + COLON + StringUtils.SPACE + split[2].trim() + "\n" + split[0];
+ } else {
+ return errorMsg;
+ }
+ }
+
+ public static String addLimit(String originString) {
+ if (StringUtils.isBlank(originString)) {
+ return originString;
+ }
+ String replacedString = originString.trim();
+ Matcher selectMatcher = SELECT_PATTERN.matcher(replacedString);
+ if (!selectMatcher.find()) {
+ return originString;
+ }
+
+ while (replacedString.endsWith(SEMI_COLON)) {
+ replacedString = replacedString.substring(0, replacedString.length() - 1).trim();
+ }
+
+ Matcher limitMatcher = LIMIT_PATTERN.matcher(replacedString);
+ return limitMatcher.find() ? originString : replacedString.concat(" limit 1");
+ }
+
public static String massageExpression(NDataModel model, String project, String expression,
- QueryContext.AclInfo aclInfo, boolean massageToPushdown) {
+ QueryContext.AclInfo aclInfo, boolean isForPushDown) {
String tempConst = "'" + RandomUtil.randomUUIDStr() + "'";
StringBuilder forCC = new StringBuilder();
- forCC.append("select ");
- forCC.append(expression);
- forCC.append(" ,").append(tempConst);
- forCC.append(" ");
+ forCC.append("select ").append(expression).append(" ,").append(tempConst) //
+ .append(" FROM ") //
+ .append(model.getRootFactTable().getTableDesc().getDoubleQuoteIdentity());
appendJoinStatement(model, forCC, false);
String ccSql = KeywordDefaultDirtyHack.transform(forCC.toString());
@@ -99,10 +202,10 @@ public class KapQueryUtil {
modelMap.put(model.getUuid(), model);
ccSql = RestoreFromComputedColumn.convertWithGivenModels(ccSql, project, DEFAULT_SCHEMA, modelMap);
QueryParams queryParams = new QueryParams(project, ccSql, DEFAULT_SCHEMA, false);
- queryParams.setKylinConfig(getKylinConfig(project));
+ queryParams.setKylinConfig(NProjectManager.getProjectConfig(project));
queryParams.setAclInfo(aclInfo);
- if (massageToPushdown) {
+ if (isForPushDown) {
ccSql = massagePushDownSql(queryParams);
}
} catch (Exception e) {
@@ -125,12 +228,7 @@ public class KapQueryUtil {
public static void appendJoinStatement(NDataModel model, StringBuilder sql, boolean singleLine) {
final String sep = singleLine ? " " : "\n";
Set<TableRef> dimTableCache = Sets.newHashSet();
-
- TableRef rootTable = model.getRootFactTable();
- sql.append(String.format(Locale.ROOT, "FROM \"%s\".\"%s\" as \"%s\"", rootTable.getTableDesc().getDatabase(),
- rootTable.getTableDesc().getName(), rootTable.getAlias()));
sql.append(sep);
-
for (JoinTableDesc lookupDesc : model.getJoinTables()) {
JoinDesc join = lookupDesc.getJoin();
TableRef dimTable = lookupDesc.getTableRef();
@@ -154,19 +252,14 @@ public class KapQueryUtil {
if (pk.length == 0 && join.getNonEquiJoinCondition() != null) {
sql.append(join.getNonEquiJoinCondition().getExpr());
dimTableCache.add(dimTable);
- continue;
- }
-
- for (int i = 0; i < pk.length; i++) {
- if (i > 0) {
- sql.append(" AND ");
- }
- sql.append(String.format(Locale.ROOT, "%s = %s", fk[i].getDoubleQuoteExpressionInSourceDB(),
- pk[i].getDoubleQuoteExpressionInSourceDB()));
+ } else {
+ String collect = IntStream.range(0, pk.length) //
+ .mapToObj(i -> fk[i].getDoubleQuoteExpressionInSourceDB() + " = "
+ + pk[i].getDoubleQuoteExpressionInSourceDB())
+ .collect(Collectors.joining(" AND ", "", sep));
+ sql.append(collect);
+ dimTableCache.add(dimTable);
}
- sql.append(sep);
-
- dimTableCache.add(dimTable);
}
}
@@ -206,10 +299,7 @@ public class KapQueryUtil {
if (!isContainAggregate(joinLeftChild) && !isContainAggregate(joinRightChild)) {
return false;
}
- if (isContainAggregate(joinLeftChild) && isContainAggregate(joinRightChild)) {
- return false;
- }
- return true;
+ return !isContainAggregate(joinLeftChild) || !isContainAggregate(joinRightChild);
}
private static boolean isContainAggregate(RelNode node) {
@@ -275,10 +365,7 @@ public class KapQueryUtil {
}
if (SqlKind.CAST == rexNode.getKind()) {
RexNode operand = ((RexCall) rexNode).getOperands().get(0);
- if (operand instanceof RexCall && operand.getKind() != SqlKind.CASE) {
- return false;
- }
- return true;
+ return !(operand instanceof RexCall) || operand.getKind() == SqlKind.CASE;
}
return false;
@@ -312,30 +399,27 @@ public class KapQueryUtil {
initQueryTransformersIfNeeded(queryParams.getKylinConfig(), queryParams.isCCNeeded());
String sql = queryParams.getSql();
for (IQueryTransformer t : queryTransformers) {
- if (Thread.currentThread().isInterrupted()) {
- log.error("SQL transformation is timeout and interrupted before {}", t.getClass());
- if (SlowQueryDetector.getRunningQueries().get(Thread.currentThread()).isStopByUser()) {
- throw new UserStopQueryException("");
- }
- QueryContext.current().getQueryTagInfo().setTimeout(true);
- throw new KylinTimeoutException("The query exceeds the set time limit of "
- + KylinConfig.getInstanceFromEnv().getQueryTimeoutSeconds()
- + "s. Current step: SQL transformation. ");
- }
+ QueryUtil.checkThreadInterrupted("Interrupted sql transformation at the stage of " + t.getClass(),
+ "Current step: SQL transformation.");
sql = t.transform(sql, queryParams.getProject(), queryParams.getDefaultSchema());
}
return sql;
}
+ private static String trimRightSemiColon(String sql) {
+ while (sql.endsWith(SEMI_COLON)) {
+ sql = sql.substring(0, sql.length() - 1).trim();
+ }
+ return sql;
+ }
+
public static String normalMassageSql(KylinConfig kylinConfig, String sql, int limit, int offset) {
sql = sql.trim();
- sql = sql.replace("\r", " ").replace("\n", System.getProperty("line.separator"));
-
- while (sql.endsWith(";"))
- sql = sql.substring(0, sql.length() - 1);
+ sql = sql.replace("\r", StringUtils.SPACE).replace("\n", System.getProperty("line.separator"));
+ sql = trimRightSemiColon(sql);
//Split keywords and variables from sql by punctuation and whitespace character
- List<String> sqlElements = Lists.newArrayList(sql.toLowerCase(Locale.ROOT).split("(?![\\._\'\"`])\\p{P}|\\s+"));
+ List<String> sqlElements = Lists.newArrayList(sql.toLowerCase(Locale.ROOT).split("(?![._'\"`])\\p{P}|\\s+"));
Integer maxRows = kylinConfig.getMaxResultRows();
if (maxRows != null && maxRows > 0 && (maxRows < limit || limit <= 0)) {
@@ -343,14 +427,14 @@ public class KapQueryUtil {
}
// https://issues.apache.org/jira/browse/KYLIN-2649
- if (kylinConfig.getForceLimit() > 0 && limit <=0 && !sql.toLowerCase(Locale.ROOT).contains("limit")
- && sql.toLowerCase(Locale.ROOT).matches("^select\\s+\\*\\p{all}*")) {
+ if (kylinConfig.getForceLimit() > 0 && limit <= 0 && !sql.toLowerCase(Locale.ROOT).contains("limit")
+ && SELECT_STAR_PTN.matcher(sql).find()) {
limit = kylinConfig.getForceLimit();
}
if (checkBigQueryPushDown(kylinConfig)) {
long bigQueryThreshold = BigQueryThresholdUpdater.getBigQueryThreshold();
- if (limit <=0 && bigQueryThreshold > 0) {
+ if (limit <= 0 && bigQueryThreshold > 0) {
log.info("Big query route to pushdown, Add limit {} to sql.", bigQueryThreshold);
limit = (int) bigQueryThreshold;
}
@@ -368,7 +452,8 @@ public class KapQueryUtil {
}
public static boolean checkBigQueryPushDown(KylinConfig kylinConfig) {
- return kylinConfig.isBigQueryPushDown() && JDBC.equals(KapConfig.getInstanceFromEnv().getShareStateSwitchImplement());
+ return kylinConfig.isBigQueryPushDown()
+ && JDBC.equals(KapConfig.getInstanceFromEnv().getShareStateSwitchImplement());
}
public static void initQueryTransformersIfNeeded(KylinConfig kylinConfig, boolean isCCNeeded) {
@@ -389,7 +474,7 @@ public class KapQueryUtil {
public static List<IQueryTransformer> initTransformers(boolean isCCNeeded, String[] configTransformers) {
List<IQueryTransformer> transformers = Lists.newArrayList();
- List<String> classList = io.kyligence.kap.guava20.shaded.common.collect.Lists.newArrayList(configTransformers);
+ List<String> classList = Lists.newArrayList(configTransformers);
classList.removeIf(clazz -> {
String name = clazz.substring(clazz.lastIndexOf(".") + 1);
return REMOVED_TRANSFORMERS.contains(name);
@@ -410,26 +495,19 @@ public class KapQueryUtil {
return transformers;
}
+ // fix KE-34379，filter "/*+ MODEL_PRIORITY({cube_name}) */" hint
+ private static final Pattern SQL_HINT_ERASER = Pattern
+ .compile("/\\*\\s*\\+\\s*(?i)MODEL_PRIORITY\\s*\\([\\s\\S]*\\)\\s*\\*/");
+
public static String massagePushDownSql(QueryParams queryParams) {
String sql = queryParams.getSql();
- while (sql.endsWith(";"))
- sql = sql.substring(0, sql.length() - 1);
- // fix KE-34379，filter "/*+ MODEL_PRIORITY({cube_name}) */" hint
- String regex = "/\\*\\s*\\+\\s*(?i)MODEL_PRIORITY\\s*\\([\\s\\S]*\\)\\s*\\*/";
- sql = Pattern.compile(regex).matcher(sql).replaceAll("");
+ sql = trimRightSemiColon(sql);
+
+ sql = SQL_HINT_ERASER.matcher(sql).replaceAll("");
initPushDownConvertersIfNeeded(queryParams.getKylinConfig());
for (IPushDownConverter converter : pushDownConverters) {
- if (Thread.currentThread().isInterrupted()) {
- log.error("Push-down SQL conver transformation is timeout and interrupted before {}",
- converter.getClass());
- if (SlowQueryDetector.getRunningQueries().get(Thread.currentThread()).isStopByUser()) {
- throw new UserStopQueryException("");
- }
- QueryContext.current().getQueryTagInfo().setTimeout(true);
- throw new KylinTimeoutException("The query exceeds the set time limit of "
- + KylinConfig.getInstanceFromEnv().getQueryTimeoutSeconds()
- + "s. Current step: Massage push-down sql. ");
- }
+ QueryUtil.checkThreadInterrupted("Interrupted sql transformation at the stage of " + converter.getClass(),
+ "Current step: Massage push-down sql. ");
sql = converter.convert(sql, queryParams.getProject(), queryParams.getDefaultSchema());
}
return sql;
@@ -457,13 +535,15 @@ public class KapQueryUtil {
pushDownConverters = Collections.unmodifiableList(converters);
}
- public static KylinConfig getKylinConfig(String project) {
- NProjectManager projectManager = NProjectManager.getInstance(KylinConfig.getInstanceFromEnv());
- ProjectInstance projectInstance = projectManager.getProject(project);
- return projectInstance.getConfig();
- }
-
- public interface IQueryTransformer {
- String transform(String sql, String project, String defaultSchema);
+ public static void checkThreadInterrupted(String errorMsgLog, String stepInfo) {
+ if (Thread.currentThread().isInterrupted()) {
+ log.error("{} {}", QueryContext.current().getQueryId(), errorMsgLog);
+ if (SlowQueryDetector.getRunningQueries().get(Thread.currentThread()).isStopByUser()) {
+ throw new UserStopQueryException("");
+ }
+ QueryContext.current().getQueryTagInfo().setTimeout(true);
+ throw new KylinTimeoutException("The query exceeds the set time limit of "
+ + KylinConfig.getInstanceFromEnv().getQueryTimeoutSeconds() + "s. " + stepInfo);
+ }
}
}
diff --git a/src/query-common/src/main/java/org/apache/kylin/query/util/RestoreFromComputedColumn.java b/src/query-common/src/main/java/org/apache/kylin/query/util/RestoreFromComputedColumn.java
index 9d1434d72a..83d8a28a7e 100644
--- a/src/query-common/src/main/java/org/apache/kylin/query/util/RestoreFromComputedColumn.java
+++ b/src/query-common/src/main/java/org/apache/kylin/query/util/RestoreFromComputedColumn.java
@@ -34,15 +34,16 @@ import org.apache.calcite.sql.SqlSelect;
import org.apache.calcite.sql.parser.SqlParseException;
import org.apache.calcite.sql.util.SqlBasicVisitor;
import org.apache.calcite.util.Litmus;
+import org.apache.commons.collections.MapUtils;
import org.apache.kylin.common.KapConfig;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.Pair;
-import org.apache.kylin.metadata.model.tool.CalciteParser;
-import org.apache.kylin.source.adhocquery.IPushDownConverter;
import org.apache.kylin.common.util.Unsafe;
import org.apache.kylin.metadata.cube.model.NDataflowManager;
import org.apache.kylin.metadata.model.ComputedColumnDesc;
import org.apache.kylin.metadata.model.NDataModel;
+import org.apache.kylin.metadata.model.tool.CalciteParser;
+import org.apache.kylin.source.adhocquery.IPushDownConverter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -57,7 +58,10 @@ public class RestoreFromComputedColumn implements IPushDownConverter {
private static final Logger logger = LoggerFactory.getLogger(RestoreFromComputedColumn.class);
public static String convertWithGivenModels(String sql, String project, String defaultSchema,
- Map<String, NDataModel> dataModelDescs) throws SqlParseException {
+ Map<String, NDataModel> idToModelMap) throws SqlParseException {
+ if (MapUtils.isEmpty(idToModelMap)) {
+ return sql;
+ }
List<SqlCall> selectOrOrderbys = SqlSubqueryFinder.getSubqueries(sql, true);
@@ -76,13 +80,14 @@ public class RestoreFromComputedColumn implements IPushDownConverter {
int recursionTimes = 0;
int maxRecursionTimes = KapConfig.getInstanceFromEnv().getComputedColumnMaxRecursionTimes();
- while ((recursionTimes++) < maxRecursionTimes) {
-
+ while (recursionTimes < maxRecursionTimes) {
+ QueryUtil.checkThreadInterrupted("Interrupted sql transformation at the stage of RestoreFromComputedColumn",
+ "Current step: SQL transformation");
+ recursionTimes++;
boolean recursionCompleted = true;
-
for (int i = 0; i < selectOrOrderbys.size(); i++) { //subquery will precede
Pair<String, Integer> choiceForCurrentSubquery = restoreComputedColumn(sql, selectOrOrderbys.get(i),
- topColumns.get(i), dataModelDescs, queryAliasMatcher);
+ topColumns.get(i), idToModelMap, queryAliasMatcher);
if (choiceForCurrentSubquery != null) {
sql = choiceForCurrentSubquery.getFirst();
@@ -113,11 +118,6 @@ public class RestoreFromComputedColumn implements IPushDownConverter {
* check whether it needs parenthesis
* 1. not need if it is a top node
* 2. not need if it is in parenthesis
- *
- * @param originSql
- * @param topColumns
- * @param replaceRange
- * @return
*/
private static boolean needParenthesis(String originSql, List<SqlNode> topColumns, ReplaceRange replaceRange) {
if (replaceRange.addAlias) {
@@ -170,28 +170,28 @@ public class RestoreFromComputedColumn implements IPushDownConverter {
}
static Pair<String, Integer> restoreComputedColumn(String sql, SqlCall selectOrOrderby, List<SqlNode> topColumns,
- Map<String, NDataModel> dataModelDescs, QueryAliasMatcher queryAliasMatcher) throws SqlParseException {
- SqlSelect sqlSelect = KapQueryUtil.extractSqlSelect(selectOrOrderby);
+ Map<String, NDataModel> modelMap, QueryAliasMatcher queryAliasMatcher) {
+ SqlSelect sqlSelect = QueryUtil.extractSqlSelect(selectOrOrderby);
if (sqlSelect == null)
return Pair.newPair(sql, 0);
Pair<String, Integer> choiceForCurrentSubquery = null; //<new sql, number of changes by the model>
//give each data model a chance to rewrite, choose the model that generates most changes
- for (NDataModel modelDesc : dataModelDescs.values()) {
-
- QueryAliasMatchInfo info = queryAliasMatcher.match(modelDesc, sqlSelect);
+ for (NDataModel model : modelMap.values()) {
+ QueryAliasMatchInfo info = model.getComputedColumnDescs().isEmpty() ? null
+ : queryAliasMatcher.match(model, sqlSelect);
+ QueryUtil.checkThreadInterrupted("Interrupted sql transformation at the stage of RestoreFromComputedColumn",
+ "Current step: SQL transformation");
if (info == null) {
continue;
}
- Pair<String, Integer> ret = restoreComputedColumn(sql, selectOrOrderby, topColumns, modelDesc, info);
-
- if (ret.getSecond() == 0)
- continue;
-
- if ((choiceForCurrentSubquery == null) || (ret.getSecond() > choiceForCurrentSubquery.getSecond())) {
+ Pair<String, Integer> ret = restoreComputedColumn(sql, selectOrOrderby, topColumns, model, info);
+ if (ret.getSecond() != 0
+ && (choiceForCurrentSubquery == null || ret.getSecond() > choiceForCurrentSubquery.getSecond())) {
choiceForCurrentSubquery = ret;
+
}
}
@@ -202,15 +202,14 @@ public class RestoreFromComputedColumn implements IPushDownConverter {
* return the replaced sql, and the count of changes in the replaced sql
*/
static Pair<String, Integer> restoreComputedColumn(String inputSql, SqlCall selectOrOrderby,
- List<SqlNode> topColumns, NDataModel dataModelDesc, QueryAliasMatchInfo matchInfo)
- throws SqlParseException {
+ List<SqlNode> topColumns, NDataModel dataModelDesc, QueryAliasMatchInfo matchInfo) {
String result = inputSql;
Set<String> ccColNamesWithPrefix = Sets.newHashSet();
ccColNamesWithPrefix.addAll(dataModelDesc.getComputedColumnNames());
List<ColumnUsage> columnUsages = ColumnUsagesFinder.getColumnUsages(selectOrOrderby, ccColNamesWithPrefix);
- if (columnUsages.size() == 0) {
+ if (columnUsages.isEmpty()) {
return Pair.newPair(inputSql, 0);
}
@@ -232,10 +231,6 @@ public class RestoreFromComputedColumn implements IPushDownConverter {
//TODO cannot do this check because cc is not visible on schema without solid realizations
// after this is done, constrains on #1932 can be relaxed
- // TblColRef tblColRef = QueryAliasMatchInfo.resolveTblColRef(queryAliasMatchInfo.getQueryAlias(), columnName);
- // //for now, must be fact table
- // Preconditions.checkState(tblColRef.getTableRef().getTableIdentity()
- // .equals(dataModelDesc.getRootFactTable().getTableIdentity()));
ComputedColumnDesc computedColumnDesc = dataModelDesc
.findCCByCCColumnName(ComputedColumnDesc.getOriginCcName(columnName));
// The computed column expression is defined based on alias in model, e.g. BUYER_COUNTRY.x + BUYER_ACCOUNT.y
@@ -282,34 +277,32 @@ public class RestoreFromComputedColumn implements IPushDownConverter {
result = Unsafe.format(Locale.ROOT, pattern, result.substring(0, toBeReplaced.beginPos),
toBeReplaced.replaceExpr, result.substring(toBeReplaced.endPos));
- last = toBeReplaced;// new Pair<>(toBeReplaced.getFirst(), new Pair<>(start, end+6));// toBeReplaced;
+ last = toBeReplaced;// new Pair<>(toBeReplaced.getFirst(), new Pair<>(start, end+6));// toBeReplaced
}
return Pair.newPair(result, toBeReplacedUsages.size());
}
@Override
- public String convert(String originSql, String project, String defaultSchema) {
+ public String convert(String sql, String project, String defaultSchema) {
try {
- String sql = originSql;
if (project == null || sql == null) {
return sql;
}
- Map<String, NDataModel> dataModelDescs = new LinkedHashMap<>();
-
+ Map<String, NDataModel> modelMap = new LinkedHashMap<>();
NDataflowManager dataflowManager = NDataflowManager.getInstance(KylinConfig.getInstanceFromEnv(), project);
for (NDataModel modelDesc : dataflowManager.listUnderliningDataModels()) {
- dataModelDescs.put(modelDesc.getUuid(), modelDesc);
+ modelMap.put(modelDesc.getUuid(), modelDesc);
}
- return convertWithGivenModels(sql, project, defaultSchema, dataModelDescs);
+ return convertWithGivenModels(sql, project, defaultSchema, modelMap);
} catch (Exception e) {
logger.debug(
"Something unexpected while RestoreFromComputedColumn transforming the query, return original query",
e);
- return originSql;
+ return sql;
}
}
@@ -335,8 +328,7 @@ public class RestoreFromComputedColumn implements IPushDownConverter {
this.usages = Lists.newArrayList();
}
- public static List<ColumnUsage> getColumnUsages(SqlCall selectOrOrderby, Set<String> columnNames)
- throws SqlParseException {
+ public static List<ColumnUsage> getColumnUsages(SqlCall selectOrOrderby, Set<String> columnNames) {
ColumnUsagesFinder sqlSubqueryFinder = new ColumnUsagesFinder(columnNames);
selectOrOrderby.accept(sqlSubqueryFinder);
return sqlSubqueryFinder.getUsages();
@@ -397,7 +389,7 @@ public class RestoreFromComputedColumn implements IPushDownConverter {
}
}
- static private class ColumnUsage {
+ private static class ColumnUsage {
SqlIdentifier sqlIdentifier;
boolean addAlias;
diff --git a/src/query-common/src/main/java/org/apache/kylin/query/util/WithToSubQueryTransformer.java b/src/query-common/src/main/java/org/apache/kylin/query/util/WithToSubQueryTransformer.java
index 7d4f5e643b..2e064608e9 100644
--- a/src/query-common/src/main/java/org/apache/kylin/query/util/WithToSubQueryTransformer.java
+++ b/src/query-common/src/main/java/org/apache/kylin/query/util/WithToSubQueryTransformer.java
@@ -42,11 +42,11 @@ import org.apache.calcite.sql.parser.SqlParseException;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.Pair;
import org.apache.kylin.metadata.model.tool.CalciteParser;
+import org.apache.kylin.metadata.project.NProjectManager;
+import org.apache.kylin.query.IQueryTransformer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.kylin.metadata.project.NProjectManager;
-
/**
* Transform "WITH AS ... SELECT" SQL to SQL with subquery
*
@@ -62,7 +62,7 @@ import org.apache.kylin.metadata.project.NProjectManager;
* So the preparedStatement parameters should also be transformed
*
*/
-public class WithToSubQueryTransformer implements KapQueryUtil.IQueryTransformer {
+public class WithToSubQueryTransformer implements IQueryTransformer {
private static final Logger logger = LoggerFactory.getLogger(WithToSubQueryTransformer.class);
@Override
diff --git a/src/query-server/src/test/java/org/apache/kylin/rest/controller/NQueryControllerTest.java b/src/query-server/src/test/java/org/apache/kylin/rest/controller/NQueryControllerTest.java
index 162d274c27..4307b8168f 100644
--- a/src/query-server/src/test/java/org/apache/kylin/rest/controller/NQueryControllerTest.java
+++ b/src/query-server/src/test/java/org/apache/kylin/rest/controller/NQueryControllerTest.java
@@ -24,7 +24,7 @@ import static org.hamcrest.CoreMatchers.containsString;
import java.lang.reflect.Method;
import java.util.ArrayList;
-import java.util.Arrays;
+import java.util.Collections;
import java.util.HashMap;
import java.util.List;
@@ -40,7 +40,6 @@ import org.apache.kylin.metadata.query.NativeQueryRealization;
import org.apache.kylin.metadata.query.QueryHistory;
import org.apache.kylin.metadata.query.QueryHistoryInfo;
import org.apache.kylin.metadata.query.QueryHistoryRequest;
-import org.apache.kylin.query.util.QueryUtil;
import org.apache.kylin.rest.constant.Constant;
import org.apache.kylin.rest.model.Query;
import org.apache.kylin.rest.request.PrepareSqlRequest;
@@ -89,7 +88,7 @@ public class NQueryControllerTest extends NLocalFileMetadataTestCase {
@Before
public void setup() {
- MockitoAnnotations.initMocks(this);
+ MockitoAnnotations.openMocks(this);
mockMvc = MockMvcBuilders.standaloneSetup(nQueryController).defaultRequest(MockMvcRequestBuilders.get("/"))
.build();
@@ -124,7 +123,7 @@ public class NQueryControllerTest extends NLocalFileMetadataTestCase {
.accept(MediaType.parseMediaType(HTTP_VND_APACHE_KYLIN_JSON)))
.andExpect(MockMvcResultMatchers.status().isOk());
- Mockito.verify(nQueryController).query((PrepareSqlRequest) Mockito.any(), Mockito.anyString());
+ Mockito.verify(nQueryController).query(Mockito.any(), Mockito.anyString());
}
@Test
@@ -136,12 +135,11 @@ public class NQueryControllerTest extends NLocalFileMetadataTestCase {
sql.setForcedToIndex(true);
sql.setForcedToPushDown(false);
mockMvc.perform(MockMvcRequestBuilders.post("/api/query").contentType(MediaType.APPLICATION_JSON)
- .content(JsonUtil.writeValueAsString(sql))
- .header("User-Agent", "Chrome/89.0.4389.82 Safari/537.36")
+ .content(JsonUtil.writeValueAsString(sql)).header("User-Agent", "Chrome/89.0.4389.82 Safari/537.36")
.accept(MediaType.parseMediaType(HTTP_VND_APACHE_KYLIN_JSON)))
.andExpect(MockMvcResultMatchers.status().isOk());
- Mockito.verify(nQueryController).query((PrepareSqlRequest) Mockito.any(), Mockito.anyString());
+ Mockito.verify(nQueryController).query(Mockito.any(), Mockito.anyString());
}
@Test
@@ -154,11 +152,13 @@ public class NQueryControllerTest extends NLocalFileMetadataTestCase {
PrepareSqlRequest sql = new PrepareSqlRequest();
sql.setForcedToIndex(true);
sql.setForcedToPushDown(true);
- try{
- checkForcedToParams.invoke(qc, (Object)sql);
+ try {
+ checkForcedToParams.invoke(qc, sql);
} catch (Exception e) {
- Assert.assertSame(new KylinException(
- QueryErrorCode.INVALID_QUERY_PARAMS, MsgPicker.getMsg().getCannotForceToBothPushdodwnAndIndex()).getMessage(), e.getCause().getMessage());
+ Assert.assertSame(
+ new KylinException(QueryErrorCode.INVALID_QUERY_PARAMS,
+ MsgPicker.getMsg().getCannotForceToBothPushdodwnAndIndex()).getMessage(),
+ e.getCause().getMessage());
catched = true;
}
Assert.assertTrue(catched);
@@ -167,17 +167,19 @@ public class NQueryControllerTest extends NLocalFileMetadataTestCase {
sql.setForcedToIndex(true);
sql.setForcedToPushDown(false);
sql.setForcedToTieredStorage(ForceToTieredStorage.CH_FAIL_TO_PUSH_DOWN.ordinal());
- checkForcedToParams.invoke(qc, (Object)sql);
+ checkForcedToParams.invoke(qc, sql);
catched = false;
sql = new PrepareSqlRequest();
sql.setForcedToTieredStorage(4);
- try{
- checkForcedToParams.invoke(qc, (Object)sql);
+ try {
+ checkForcedToParams.invoke(qc, sql);
} catch (Exception e) {
- Assert.assertSame(new KylinException(
- QueryErrorCode.FORCED_TO_TIEREDSTORAGE_INVALID_PARAMETER, MsgPicker.getMsg().getForcedToTieredstorageInvalidParameter()).getMessage(), e.getCause().getMessage());
+ Assert.assertSame(
+ new KylinException(QueryErrorCode.FORCED_TO_TIEREDSTORAGE_INVALID_PARAMETER,
+ MsgPicker.getMsg().getForcedToTieredstorageInvalidParameter()).getMessage(),
+ e.getCause().getMessage());
catched = true;
}
Assert.assertTrue(catched);
@@ -186,11 +188,13 @@ public class NQueryControllerTest extends NLocalFileMetadataTestCase {
sql = new PrepareSqlRequest();
sql.setForcedToTieredStorage(-1);
- try{
- checkForcedToParams.invoke(qc, (Object)sql);
+ try {
+ checkForcedToParams.invoke(qc, sql);
} catch (Exception e) {
- Assert.assertSame(new KylinException(
- QueryErrorCode.FORCED_TO_TIEREDSTORAGE_INVALID_PARAMETER, MsgPicker.getMsg().getForcedToTieredstorageInvalidParameter()).getMessage(), e.getCause().getMessage());
+ Assert.assertSame(
+ new KylinException(QueryErrorCode.FORCED_TO_TIEREDSTORAGE_INVALID_PARAMETER,
+ MsgPicker.getMsg().getForcedToTieredstorageInvalidParameter()).getMessage(),
+ e.getCause().getMessage());
catched = true;
}
Assert.assertTrue(catched);
@@ -199,7 +203,7 @@ public class NQueryControllerTest extends NLocalFileMetadataTestCase {
Mockito.when(sql.getForcedToTieredStorage()).thenThrow(new NullPointerException());
sql.setForcedToIndex(false);
sql.setForcedToPushDown(false);
- checkForcedToParams.invoke(qc, (Object)sql);
+ checkForcedToParams.invoke(qc, sql);
}
@Test
@@ -209,15 +213,13 @@ public class NQueryControllerTest extends NLocalFileMetadataTestCase {
sql.setProject(PROJECT);
sql.setForcedToTieredStorage(-1);
mockMvc.perform(MockMvcRequestBuilders.post("/api/query").contentType(MediaType.APPLICATION_JSON)
- .content(JsonUtil.writeValueAsString(sql))
- .header("User-Agent", "Chrome/89.0.4389.82 Safari/537.36")
+ .content(JsonUtil.writeValueAsString(sql)).header("User-Agent", "Chrome/89.0.4389.82 Safari/537.36")
.accept(MediaType.parseMediaType(HTTP_VND_APACHE_KYLIN_JSON)))
.andExpect(MockMvcResultMatchers.status().is5xxServerError());
- Mockito.verify(nQueryController).query((PrepareSqlRequest) Mockito.any(), Mockito.anyString());
+ Mockito.verify(nQueryController).query(Mockito.any(), Mockito.anyString());
}
-
@Test
public void testStopQuery() throws Exception {
mockMvc.perform(MockMvcRequestBuilders.delete("/api/query/1").contentType(MediaType.APPLICATION_JSON)
@@ -298,7 +300,7 @@ public class NQueryControllerTest extends NLocalFileMetadataTestCase {
.accept(MediaType.parseMediaType(HTTP_VND_APACHE_KYLIN_JSON)))
.andExpect(MockMvcResultMatchers.status().isOk());
- Mockito.verify(nQueryController).prepareQuery((PrepareSqlRequest) Mockito.any());
+ Mockito.verify(nQueryController).prepareQuery(Mockito.any());
}
@Test
@@ -412,15 +414,6 @@ public class NQueryControllerTest extends NLocalFileMetadataTestCase {
Mockito.verify(nQueryController).getMetadata("default", null);
}
- @Test
- public void testErrorMsg() {
- String errorMsg = "Error while executing SQL \"select lkp.clsfd_ga_prfl_id, ga.sum_dt, sum(ga.bounces) as bounces, sum(ga.exits) as exits, sum(ga.entrances) as entrances, sum(ga.pageviews) as pageviews, count(distinct ga.GA_VSTR_ID, ga.GA_VST_ID) as visits, count(distinct ga.GA_VSTR_ID) as uniqVistors from CLSFD_GA_PGTYPE_CATEG_LOC ga left join clsfd_ga_prfl_lkp lkp on ga.SRC_GA_PRFL_ID = lkp.SRC_GA_PRFL_ID group by lkp.clsfd_ga_prfl_id,ga.sum_dt order by lkp.clsfd_ga_prfl_id,ga. [...]
- Assert.assertEquals(
- "From line 14, column 14 to line 14, column 29: Column 'CLSFD_GA_PRFL_ID' not found in table 'LKP'\n"
- + "while executing SQL: \"select lkp.clsfd_ga_prfl_id, ga.sum_dt, sum(ga.bounces) as bounces, sum(ga.exits) as exits, sum(ga.entrances) as entrances, sum(ga.pageviews) as pageviews, count(distinct ga.GA_VSTR_ID, ga.GA_VST_ID) as visits, count(distinct ga.GA_VSTR_ID) as uniqVistors from CLSFD_GA_PGTYPE_CATEG_LOC ga left join clsfd_ga_prfl_lkp lkp on ga.SRC_GA_PRFL_ID = lkp.SRC_GA_PRFL_ID group by lkp.clsfd_ga_prfl_id,ga.sum_dt order by lkp.clsfd_ga_prfl_id,ga.sum_d [...]
- QueryUtil.makeErrorMsgUserFriendly(errorMsg));
- }
-
@Test
public void testQueryStatistics() throws Exception {
mockMvc.perform(MockMvcRequestBuilders.get("/api/query/statistics").contentType(MediaType.APPLICATION_JSON)
@@ -474,16 +467,6 @@ public class NQueryControllerTest extends NLocalFileMetadataTestCase {
return queries;
}
- private List<String> mockQueryHistorySubmitters() {
- final List<String> submitters = Lists.newArrayList();
- submitters.add("ADMIN");
- submitters.add("USER1");
- submitters.add("USER2");
- submitters.add("USER3");
- submitters.add("USER4");
- return submitters;
- }
-
@Test
public void testGetQueryHistories() throws Exception {
QueryHistoryRequest request = new QueryHistoryRequest();
@@ -493,7 +476,7 @@ public class NQueryControllerTest extends NLocalFileMetadataTestCase {
request.setLatencyFrom("0");
request.setLatencyTo("10");
request.setSubmitterExactlyMatch(true);
- request.setQueryStatus(Arrays.asList("FAILED"));
+ request.setQueryStatus(Collections.singletonList("FAILED"));
HashMap<String, Object> data = Maps.newHashMap();
data.put("query_histories", mockedQueryHistories());
data.put("size", 6);
diff --git a/src/query-service/src/main/java/org/apache/kylin/rest/service/QueryCacheManager.java b/src/query-service/src/main/java/org/apache/kylin/rest/service/QueryCacheManager.java
index 6f7ecff94b..bdbce8d893 100644
--- a/src/query-service/src/main/java/org/apache/kylin/rest/service/QueryCacheManager.java
+++ b/src/query-service/src/main/java/org/apache/kylin/rest/service/QueryCacheManager.java
@@ -193,9 +193,8 @@ public class QueryCacheManager implements CommonQueryCacheSupporter {
String project = sqlRequest.getProject();
for (NativeQueryRealization nativeQueryRealization : realizations) {
val modelId = nativeQueryRealization.getModelId();
- val dataflow = NDataflowManager.getInstance(KylinConfig.getInstanceFromEnv(), project)
- .getDataflow(modelId);
- nativeQueryRealization.setModelAlias(dataflow.getFusionModelAlias());
+ val dataflow = NDataflowManager.getInstance(KylinConfig.getInstanceFromEnv(), project).getDataflow(modelId);
+ nativeQueryRealization.setModelAlias(dataflow.getModelAlias());
}
return cached;
diff --git a/src/query-service/src/main/java/org/apache/kylin/rest/service/QueryService.java b/src/query-service/src/main/java/org/apache/kylin/rest/service/QueryService.java
index b1361132f9..dcc26d6a51 100644
--- a/src/query-service/src/main/java/org/apache/kylin/rest/service/QueryService.java
+++ b/src/query-service/src/main/java/org/apache/kylin/rest/service/QueryService.java
@@ -92,7 +92,6 @@ import org.apache.kylin.metadata.model.TableRef;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.model.tool.CalciteParser;
import org.apache.kylin.metadata.project.NProjectManager;
-import org.apache.kylin.metadata.project.ProjectInstance;
import org.apache.kylin.metadata.query.BigQueryThresholdUpdater;
import org.apache.kylin.metadata.query.NativeQueryRealization;
import org.apache.kylin.metadata.query.QueryHistory;
@@ -169,7 +168,6 @@ import com.google.common.collect.Maps;
import com.google.common.collect.SetMultimap;
import com.google.gson.Gson;
-import org.apache.kylin.query.util.KapQueryUtil;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.Getter;
@@ -232,9 +230,7 @@ public class QueryService extends BasicService implements CacheSignatureQuerySup
try {
//project level config
- ProjectInstance projectInstance = NProjectManager.getInstance(KylinConfig.getInstanceFromEnv())
- .getProject(project);
- api = projectInstance.getConfig().getProjectForcedToTieredStorage();
+ api = NProjectManager.getProjectConfig(project).getProjectForcedToTieredStorage();
switch (api) {
case CH_FAIL_TO_DFS:
case CH_FAIL_TO_PUSH_DOWN:
@@ -268,7 +264,7 @@ public class QueryService extends BasicService implements CacheSignatureQuerySup
slowQueryDetector.queryStart(sqlRequest.getStopId());
markHighPriorityQueryIfNeeded();
- QueryParams queryParams = new QueryParams(KapQueryUtil.getKylinConfig(sqlRequest.getProject()),
+ QueryParams queryParams = new QueryParams(NProjectManager.getProjectConfig(sqlRequest.getProject()),
sqlRequest.getSql(), sqlRequest.getProject(), sqlRequest.getLimit(), sqlRequest.getOffset(), true,
sqlRequest.getExecuteAs());
queryParams.setForcedToPushDown(sqlRequest.isForcedToPushDown());
@@ -300,11 +296,8 @@ public class QueryService extends BasicService implements CacheSignatureQuerySup
QueryContext.current().setForcedToTieredStorage(enumForcedToTieredStorage);
QueryContext.current().setForceTableIndex(queryParams.isForcedToIndex());
- KylinConfig projectConfig = NProjectManager.getInstance(KylinConfig.getInstanceFromEnv())
- .getProject(queryParams.getProject()).getConfig();
-
if (QueryContext.current().getQueryTagInfo().isAsyncQuery()
- && projectConfig.isUniqueAsyncQueryYarnQueue()) {
+ && NProjectManager.getProjectConfig(sqlRequest.getProject()).isUniqueAsyncQueryYarnQueue()) {
if (StringUtils.isNotEmpty(sqlRequest.getSparkQueue())) {
queryParams.setSparkQueue(sqlRequest.getSparkQueue());
}
@@ -428,9 +421,6 @@ public class QueryService extends BasicService implements CacheSignatureQuerySup
.collect(Collectors.toList());
}
- KylinConfig projectConfig = NProjectManager.getInstance(KylinConfig.getInstanceFromEnv())
- .getProject(request.getProject()).getConfig();
-
String errorMsg = response.getExceptionMessage();
if (StringUtils.isNotBlank(errorMsg)) {
int maxLength = 5000;
@@ -466,7 +456,8 @@ public class QueryService extends BasicService implements CacheSignatureQuerySup
.put(LogReport.SCAN_FILE_COUNT, QueryContext.current().getMetrics().getFileCount())
.put(LogReport.REFUSE, response.isRefused());
String log = report.oldStyleLog();
- if (!(QueryContext.current().getQueryTagInfo().isAsyncQuery() && projectConfig.isUniqueAsyncQueryYarnQueue())) {
+ if (!(QueryContext.current().getQueryTagInfo().isAsyncQuery()
+ && NProjectManager.getProjectConfig(request.getProject()).isUniqueAsyncQueryYarnQueue())) {
logger.info(log);
logger.debug(report.jsonStyleLog());
if (request.getExecuteAs() != null)
@@ -488,12 +479,11 @@ public class QueryService extends BasicService implements CacheSignatureQuerySup
queryContext.setQueryId(UUID.fromString(sqlRequest.getQueryId()).toString());
}
try (SetThreadName ignored = new SetThreadName("Query %s", queryContext.getQueryId());
- SetLogCategory ignored2 = new SetLogCategory("query")) {
- if (sqlRequest.getExecuteAs() != null) {
+ SetLogCategory ignored2 = new SetLogCategory("query")) {
+ if (sqlRequest.getExecuteAs() != null)
sqlRequest.setUsername(sqlRequest.getExecuteAs());
- } else {
+ else
sqlRequest.setUsername(getUsername());
- }
QueryLimiter.tryAcquire();
SQLResponse response = doQueryWithCache(sqlRequest);
response.setTraces(QueryContext.currentTrace().spans().stream().map(span -> {
@@ -550,8 +540,7 @@ public class QueryService extends BasicService implements CacheSignatureQuerySup
throw new KylinException(JOB_NODE_QUERY_API_INVALID);
}
checkSqlRequestProject(sqlRequest, msg);
- final NProjectManager projectMgr = NProjectManager.getInstance(kylinConfig);
- if (projectMgr.getProject(sqlRequest.getProject()) == null) {
+ if (NProjectManager.getInstance(kylinConfig).getProject(sqlRequest.getProject()) == null) {
throw new KylinException(PROJECT_NOT_EXIST, sqlRequest.getProject());
}
if (StringUtils.isBlank(sqlRequest.getSql())) {
@@ -581,11 +570,9 @@ public class QueryService extends BasicService implements CacheSignatureQuerySup
queryContext.setAclInfo(getExecuteAclInfo(project, sqlRequest.getExecuteAs()));
QueryContext.currentTrace().startSpan(QueryTrace.SQL_TRANSFORMATION);
queryContext.getMetrics().setServer(clusterManager.getLocalServer());
+ queryContext.setProject(project);
- KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
- ProjectInstance projectInstance = NProjectManager.getInstance(kylinConfig).getProject(project);
- kylinConfig = projectInstance.getConfig();
-
+ KylinConfig kylinConfig = NProjectManager.getProjectConfig(project);
// Parsing user sql by RawSqlParser
RawSql rawSql = new RawSqlParser(sqlRequest.getSql()).parse();
rawSql.autoAppendLimit(kylinConfig, sqlRequest.getLimit(), sqlRequest.getOffset());
@@ -605,7 +592,7 @@ public class QueryService extends BasicService implements CacheSignatureQuerySup
sqlResponse = QueryUtils.handleTempStatement(sqlRequest, kylinConfig);
// search cache
- if (sqlResponse == null && isQueryCacheEnabled(kylinConfig) && !sqlRequest.isForcedToPushDown()) {
+ if (sqlResponse == null && kylinConfig.isQueryCacheEnabled() && !sqlRequest.isForcedToPushDown()) {
sqlResponse = searchCache(sqlRequest, kylinConfig);
}
@@ -678,7 +665,7 @@ public class QueryService extends BasicService implements CacheSignatureQuerySup
protected SQLResponse searchCache(SQLRequest sqlRequest, KylinConfig kylinConfig) {
SQLResponse response = searchFailedCache(sqlRequest, kylinConfig);
if (response == null) {
- response = searchSuccessCache(sqlRequest, kylinConfig);
+ response = searchSuccessCache(sqlRequest);
}
if (response != null) {
response.setDuration(0);
@@ -703,7 +690,7 @@ public class QueryService extends BasicService implements CacheSignatureQuerySup
return null;
}
- private SQLResponse searchSuccessCache(SQLRequest sqlRequest, KylinConfig kylinConfig) {
+ private SQLResponse searchSuccessCache(SQLRequest sqlRequest) {
SQLResponse response = queryCacheManager.searchSuccessCache(sqlRequest);
if (response != null) {
logger.info("The sqlResponse is found in SUCCESS_QUERY_CACHE");
@@ -713,10 +700,8 @@ public class QueryService extends BasicService implements CacheSignatureQuerySup
}
private void addToQueryHistory(SQLRequest sqlRequest, SQLResponse sqlResponse, String originalSql) {
- KylinConfig projectKylinConfig = NProjectManager.getInstance(KylinConfig.getInstanceFromEnv())
- .getProject(sqlRequest.getProject()).getConfig();
if (!(QueryContext.current().getQueryTagInfo().isAsyncQuery()
- && projectKylinConfig.isUniqueAsyncQueryYarnQueue())) {
+ && NProjectManager.getProjectConfig(sqlRequest.getProject()).isUniqueAsyncQueryYarnQueue())) {
try {
if (!sqlResponse.isPrepare() && QueryMetricsContext.isStarted()) {
val queryMetricsContext = QueryMetricsContext.collect(QueryContext.current());
@@ -890,7 +875,7 @@ public class QueryService extends BasicService implements CacheSignatureQuerySup
if (null == queryCacheManager.getFromExceptionCache(sqlRequest)) {
return;
}
- if (!queryCacheManager.getCache().remove(QueryCacheManager.Type.EXCEPTION_QUERY_CACHE.rootCacheName,
+ if (!queryCacheManager.getCache().remove(CommonQueryCacheSupporter.Type.EXCEPTION_QUERY_CACHE.rootCacheName,
sqlRequest.getProject(), sqlRequest.getCacheKey())) {
logger.info("Remove cache failed");
}
@@ -950,9 +935,7 @@ public class QueryService extends BasicService implements CacheSignatureQuerySup
}
boolean isACLDisabledOrAdmin(String project, QueryContext.AclInfo aclInfo) {
- KylinConfig projectKylinConfig = NProjectManager.getInstance(KylinConfig.getInstanceFromEnv())
- .getProject(project).getConfig();
- if (!projectKylinConfig.isAclTCREnabled()) {
+ if (!NProjectManager.getProjectConfig(project).isAclTCREnabled()) {
return true;
}
@@ -975,9 +958,7 @@ public class QueryService extends BasicService implements CacheSignatureQuerySup
public QueryExec newQueryExec(String project, String executeAs) {
QueryContext.current().setAclInfo(getExecuteAclInfo(project, executeAs));
- KylinConfig projectKylinConfig = NProjectManager.getInstance(KylinConfig.getInstanceFromEnv())
- .getProject(project).getConfig();
- return new QueryExec(project, projectKylinConfig, true);
+ return new QueryExec(project, NProjectManager.getProjectConfig(project), true);
}
protected QueryContext.AclInfo getExecuteAclInfo(String project) {
@@ -1014,8 +995,7 @@ public class QueryService extends BasicService implements CacheSignatureQuerySup
}
public List<TableMeta> getMetadata(String project) {
- KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
- if (!NProjectManager.getInstance(kylinConfig).getProject(project).getConfig().isSchemaCacheEnabled()) {
+ if (!NProjectManager.getProjectConfig(project).isSchemaCacheEnabled()) {
return doGetMetadata(project, null);
}
@@ -1050,9 +1030,7 @@ public class QueryService extends BasicService implements CacheSignatureQuerySup
List<String> targetModelColumns = getTargetModelColumns(targetModelName, models, project);
QueryContext.current().setAclInfo(getExecuteAclInfo(project));
- ProjectInstance projectInstance = NProjectManager.getInstance(KylinConfig.getInstanceFromEnv())
- .getProject(project);
- SchemaMetaData schemaMetaData = new SchemaMetaData(project, projectInstance.getConfig());
+ SchemaMetaData schemaMetaData = new SchemaMetaData(project, NProjectManager.getProjectConfig(project));
List<TableMeta> tableMetas = new LinkedList<>();
SetMultimap<String, String> tbl2ccNames = collectComputedColumns(project);
@@ -1069,17 +1047,18 @@ public class QueryService extends BasicService implements CacheSignatureQuerySup
int columnOrdinal = 1;
for (StructField field : tableSchema.getFields()) {
- ColumnMeta colmnMeta = constructColumnMeta(tableSchema, field, columnOrdinal);
+ ColumnMeta columnMeta = constructColumnMeta(tableSchema, field, columnOrdinal);
columnOrdinal++;
- if (!shouldExposeColumn(projectInstance, colmnMeta, tbl2ccNames)) {
+ if (!shouldExposeColumn(project, columnMeta, tbl2ccNames)) {
continue;
}
- if (!colmnMeta.getCOLUMN_NAME().toUpperCase(Locale.ROOT).startsWith("_KY_")
- && (targetModelColumns == null || targetModelColumns.contains(colmnMeta.getTABLE_SCHEM() + "."
- + colmnMeta.getTABLE_NAME() + "." + colmnMeta.getCOLUMN_NAME()))) {
- tblMeta.addColumn(colmnMeta);
+ String qualifiedCol = columnMeta.getTABLE_SCHEM() + "." + columnMeta.getTABLE_NAME() + "."
+ + columnMeta.getCOLUMN_NAME();
+ if (!columnMeta.getCOLUMN_NAME().toUpperCase(Locale.ROOT).startsWith("_KY_")
+ && isQualifiedColumn(targetModelColumns, qualifiedCol)) {
+ tblMeta.addColumn(columnMeta);
}
}
@@ -1088,9 +1067,12 @@ public class QueryService extends BasicService implements CacheSignatureQuerySup
return tableMetas;
}
+ private boolean isQualifiedColumn(List<String> targetModelColumns, String qualifiedCol) {
+ return targetModelColumns == null || targetModelColumns.contains(qualifiedCol);
+ }
+
public List<TableMetaWithType> getMetadataV2(String project, String modelAlias) {
- KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
- if (!NProjectManager.getInstance(kylinConfig).getProject(project).getConfig().isSchemaCacheEnabled()) {
+ if (!NProjectManager.getProjectConfig(project).isSchemaCacheEnabled()) {
return doGetMetadataV2(project, modelAlias);
}
@@ -1143,9 +1125,7 @@ public class QueryService extends BasicService implements CacheSignatureQuerySup
List<String> targetModelColumns = getTargetModelColumns(targetModelName, models, project);
QueryContext.current().setAclInfo(getExecuteAclInfo(project));
- ProjectInstance projectInstance = NProjectManager.getInstance(KylinConfig.getInstanceFromEnv())
- .getProject(project);
- SchemaMetaData schemaMetaData = new SchemaMetaData(project, projectInstance.getConfig());
+ SchemaMetaData schemaMetaData = new SchemaMetaData(project, NProjectManager.getProjectConfig(project));
Map<TableMetaIdentify, TableMetaWithType> tableMap = constructTableMeta(schemaMetaData, targetModelTables);
Map<ColumnMetaIdentify, ColumnMetaWithType> columnMap = constructTblColMeta(schemaMetaData, project,
targetModelColumns);
@@ -1166,14 +1146,16 @@ public class QueryService extends BasicService implements CacheSignatureQuerySup
List<String> targetModelColumns = null;
if (targetModelName != null) {
NIndexPlanManager indexPlanManager = getManager(NIndexPlanManager.class, project);
- targetModelColumns = !getManager(NProjectManager.class).getProject(project).getConfig()
- .exposeAllModelRelatedColumns() ? models.stream().map(model -> {
+ targetModelColumns = NProjectManager.getProjectConfig(project).exposeAllModelRelatedColumns()
+ ? models.stream()
+ .flatMap(m -> m.getEffectiveCols().values().stream()
+ .map(TblColRef::getColumnWithTableAndSchema))
+ .collect(Collectors.toList())
+ : models.stream().map(model -> {
Set<Integer> relatedColIds = indexPlanManager.getIndexPlan(model.getId()).getRelatedColIds();
return relatedColIds.stream().map(id -> model.getColRef(id).getColumnWithTableAndSchema())
.collect(Collectors.toList());
- }).flatMap(List::stream).collect(Collectors.toList())
- : models.stream().flatMap(m -> m.getEffectiveCols().values().stream()
- .map(TblColRef::getColumnWithTableAndSchema)).collect(Collectors.toList());
+ }).flatMap(List::stream).collect(Collectors.toList());
}
return targetModelColumns;
}
@@ -1181,7 +1163,7 @@ public class QueryService extends BasicService implements CacheSignatureQuerySup
private List<String> getTargetModelTables(String targetModelName, List<NDataModel> models) {
return targetModelName == null ? null
: models.stream().flatMap(m -> m.getAllTableRefs().stream().map(TableRef::getTableIdentity))
- .collect(Collectors.toList());
+ .collect(Collectors.toList());
}
private List<NDataModel> getModels(String project, String targetModelName) {
@@ -1210,8 +1192,6 @@ public class QueryService extends BasicService implements CacheSignatureQuerySup
private LinkedHashMap<ColumnMetaIdentify, ColumnMetaWithType> constructTblColMeta(SchemaMetaData schemaMetaData,
String project, List<String> targetModelColumns) {
LinkedHashMap<ColumnMetaIdentify, ColumnMetaWithType> columnMap = Maps.newLinkedHashMap();
- ProjectInstance projectInstance = NProjectManager.getInstance(KylinConfig.getInstanceFromEnv())
- .getProject(project);
SetMultimap<String, String> tbl2ccNames = collectComputedColumns(project);
for (TableSchema tableSchema : schemaMetaData.getTables()) {
@@ -1221,7 +1201,7 @@ public class QueryService extends BasicService implements CacheSignatureQuerySup
.ofColumnMeta(constructColumnMeta(tableSchema, field, columnOrdinal));
columnOrdinal++;
- if (!shouldExposeColumn(projectInstance, columnMeta, tbl2ccNames)) {
+ if (!shouldExposeColumn(project, columnMeta, tbl2ccNames)) {
continue;
}
@@ -1262,9 +1242,8 @@ public class QueryService extends BasicService implements CacheSignatureQuerySup
}
private SetMultimap<String, String> collectComputedColumns(String project) {
- NProjectManager projectManager = NProjectManager.getInstance(KylinConfig.getInstanceFromEnv());
SetMultimap<String, String> tbl2ccNames = HashMultimap.create();
- projectManager.listAllRealizations(project).forEach(rea -> {
+ getManager(NProjectManager.class).listAllRealizations(project).forEach(rea -> {
val upperCaseCcNames = rea.getModel().getComputedColumnNames().stream()
.map(str -> str.toUpperCase(Locale.ROOT)).collect(Collectors.toList());
tbl2ccNames.putAll(rea.getModel().getRootFactTable().getAlias().toUpperCase(Locale.ROOT), upperCaseCcNames);
@@ -1273,28 +1252,25 @@ public class QueryService extends BasicService implements CacheSignatureQuerySup
return tbl2ccNames;
}
- private boolean shouldExposeColumn(ProjectInstance projectInstance, ColumnMeta columnMeta,
- SetMultimap<String, String> tbl2ccNames) {
+ private boolean shouldExposeColumn(String project, ColumnMeta columnMeta, SetMultimap<String, String> tbl2ccNames) {
// check for cc exposing
// exposeComputedColumn=True, expose columns anyway
- if (projectInstance.getConfig().exposeComputedColumn()) {
+ if (NProjectManager.getProjectConfig(project).exposeComputedColumn()) {
return true;
}
// only check cc expose when exposeComputedColumn=False
// do not expose column if it is a computed column
- return !isComputedColumn(projectInstance.getName(), columnMeta.getCOLUMN_NAME().toUpperCase(Locale.ROOT),
- columnMeta.getTABLE_NAME(), tbl2ccNames);
+ return !isComputedColumn(columnMeta.getCOLUMN_NAME().toUpperCase(Locale.ROOT), columnMeta.getTABLE_NAME(),
+ tbl2ccNames);
}
/**
- * @param project
* @param ccName
* @param table only support table alias like "TEST_COUNT" or table indentity "default.TEST_COUNT"
* @return
*/
- private boolean isComputedColumn(String project, String ccName, String table,
- SetMultimap<String, String> tbl2ccNames) {
+ private boolean isComputedColumn(String ccName, String table, SetMultimap<String, String> tbl2ccNames) {
return CollectionUtils.isNotEmpty(tbl2ccNames.get(table.toUpperCase(Locale.ROOT)))
&& tbl2ccNames.get(table.toUpperCase(Locale.ROOT)).contains(ccName.toUpperCase(Locale.ROOT));
diff --git a/src/query-service/src/main/java/org/apache/kylin/rest/util/QueryUtils.java b/src/query-service/src/main/java/org/apache/kylin/rest/util/QueryUtils.java
index 9dbae864fb..a9aba9cad5 100644
--- a/src/query-service/src/main/java/org/apache/kylin/rest/util/QueryUtils.java
+++ b/src/query-service/src/main/java/org/apache/kylin/rest/util/QueryUtils.java
@@ -26,6 +26,7 @@ import org.apache.kylin.metadata.project.NProjectManager;
import org.apache.kylin.query.engine.PrepareSqlStateParam;
import org.apache.kylin.query.engine.QueryExec;
import org.apache.kylin.query.util.QueryParams;
+import org.apache.kylin.query.util.QueryUtil;
import org.apache.kylin.query.util.TempStatementUtil;
import org.apache.kylin.rest.request.PrepareSqlRequest;
import org.apache.kylin.rest.request.SQLRequest;
@@ -33,8 +34,6 @@ import org.apache.kylin.rest.response.SQLResponse;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.kylin.query.util.KapQueryUtil;
-
public class QueryUtils {
private static final Logger logger = LoggerFactory.getLogger(QueryUtils.class);
@@ -49,10 +48,8 @@ public class QueryUtils {
}
public static boolean isPrepareStatementWithParams(SQLRequest sqlRequest) {
- if (sqlRequest instanceof PrepareSqlRequest && ((PrepareSqlRequest) sqlRequest).getParams() != null
- && ((PrepareSqlRequest) sqlRequest).getParams().length > 0)
- return true;
- return false;
+ return sqlRequest instanceof PrepareSqlRequest && ((PrepareSqlRequest) sqlRequest).getParams() != null
+ && ((PrepareSqlRequest) sqlRequest).getParams().length > 0;
}
public static void fillInPrepareStatParams(SQLRequest sqlRequest, boolean pushdown) {
@@ -82,11 +79,11 @@ public class QueryUtils {
} catch (Exception e) {
logger.warn("Failed to get connection, project: {}", queryContext.getProject(), e);
}
- QueryParams queryParams = new QueryParams(KapQueryUtil.getKylinConfig(queryContext.getProject()),
- alternativeSql, queryContext.getProject(), queryContext.getLimit(),
- queryContext.getOffset(), defaultSchema, false);
+ QueryParams queryParams = new QueryParams(NProjectManager.getProjectConfig(queryContext.getProject()),
+ alternativeSql, queryContext.getProject(), queryContext.getLimit(), queryContext.getOffset(),
+ defaultSchema, false);
queryParams.setAclInfo(queryContext.getAclInfo());
- queryContext.getMetrics().setCorrectedSql(KapQueryUtil.massageSql(queryParams));
+ queryContext.getMetrics().setCorrectedSql(QueryUtil.massageSql(queryParams));
}
if (StringUtils.isEmpty(queryContext.getMetrics().getCorrectedSql())) {
queryContext.getMetrics().setCorrectedSql(alternativeSql);
diff --git a/src/query-service/src/test/java/org/apache/kylin/rest/metrics/QueryMetricsContextTest.java b/src/query-service/src/test/java/org/apache/kylin/rest/metrics/QueryMetricsContextTest.java
index 806c0298be..d19e311a2a 100644
--- a/src/query-service/src/test/java/org/apache/kylin/rest/metrics/QueryMetricsContextTest.java
+++ b/src/query-service/src/test/java/org/apache/kylin/rest/metrics/QueryMetricsContextTest.java
@@ -28,20 +28,21 @@ import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.QueryContext;
import org.apache.kylin.common.QueryTrace;
-import org.apache.kylin.metadata.realization.NoRealizationFoundException;
-import org.apache.kylin.query.exception.UserStopQueryException;
-import org.apache.kylin.query.relnode.OLAPContext;
-import org.apache.kylin.query.util.QueryParams;
import org.apache.kylin.common.util.NLocalFileMetadataTestCase;
import org.apache.kylin.metadata.cube.model.IndexEntity;
import org.apache.kylin.metadata.model.ComputedColumnDesc;
import org.apache.kylin.metadata.model.NDataModelManager;
+import org.apache.kylin.metadata.project.NProjectManager;
import org.apache.kylin.metadata.query.QueryHistory;
import org.apache.kylin.metadata.query.QueryHistoryInfo;
import org.apache.kylin.metadata.query.QueryMetrics;
import org.apache.kylin.metadata.query.QueryMetricsContext;
+import org.apache.kylin.metadata.realization.NoRealizationFoundException;
import org.apache.kylin.query.engine.QueryExec;
-import org.apache.kylin.query.util.KapQueryUtil;
+import org.apache.kylin.query.exception.UserStopQueryException;
+import org.apache.kylin.query.relnode.OLAPContext;
+import org.apache.kylin.query.util.QueryParams;
+import org.apache.kylin.query.util.QueryUtil;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@@ -81,10 +82,10 @@ public class QueryMetricsContextTest extends NLocalFileMetadataTestCase {
String defaultSchema = new QueryExec(queryContext.getProject(), KylinConfig.getInstanceFromEnv())
.getDefaultSchemaName();
- QueryParams queryParams = new QueryParams(KapQueryUtil.getKylinConfig(queryContext.getProject()),
+ QueryParams queryParams = new QueryParams(NProjectManager.getProjectConfig(queryContext.getProject()),
queryContext.getUserSQL(), queryContext.getProject(), queryContext.getLimit(), queryContext.getOffset(),
defaultSchema, false);
- return KapQueryUtil.massageSql(queryParams);
+ return QueryUtil.massageSql(queryParams);
}
@Before
diff --git a/src/query-service/src/test/java/org/apache/kylin/rest/service/ModelServiceQueryTest.java b/src/query-service/src/test/java/org/apache/kylin/rest/service/ModelServiceQueryTest.java
index 26bb854f81..c112280a1b 100644
--- a/src/query-service/src/test/java/org/apache/kylin/rest/service/ModelServiceQueryTest.java
+++ b/src/query-service/src/test/java/org/apache/kylin/rest/service/ModelServiceQueryTest.java
@@ -36,7 +36,7 @@ import org.apache.kylin.metadata.model.NDataModel;
import org.apache.kylin.metadata.model.util.ExpandableMeasureUtil;
import org.apache.kylin.metadata.query.QueryTimesResponse;
import org.apache.kylin.metadata.recommendation.candidate.JdbcRawRecStore;
-import org.apache.kylin.query.util.KapQueryUtil;
+import org.apache.kylin.query.util.QueryUtil;
import org.apache.kylin.rest.config.initialize.ModelBrokenListener;
import org.apache.kylin.rest.constant.ModelAttributeEnum;
import org.apache.kylin.rest.constant.ModelStatusToDisplayEnum;
@@ -114,7 +114,7 @@ public class ModelServiceQueryTest extends SourceTestCase {
ReflectionTestUtils.setField(semanticService, "userGroupService", userGroupService);
ReflectionTestUtils.setField(semanticService, "expandableMeasureUtil",
new ExpandableMeasureUtil((model, ccDesc) -> {
- String ccExpression = KapQueryUtil.massageComputedColumn(model, model.getProject(), ccDesc,
+ String ccExpression = QueryUtil.massageComputedColumn(model, model.getProject(), ccDesc,
AclPermissionUtil.prepareQueryContextACLInfo(model.getProject(),
semanticService.getCurrentUserGroups()));
ccDesc.setInnerExpression(ccExpression);
@@ -151,7 +151,7 @@ public class ModelServiceQueryTest extends SourceTestCase {
cleanupTestMetadata();
}
-// @Ignore("TODO: re-run to check.")
+ // @Ignore("TODO: re-run to check.")
@Test
public void testQueryModels() {
String project = "streaming_test";
@@ -233,7 +233,7 @@ public class ModelServiceQueryTest extends SourceTestCase {
Assert.assertEquals(ModelStatusToDisplayEnum.BROKEN, nDataModelResponse.getStatus());
}
-// @Ignore("TODO: re-run to check.")
+ // @Ignore("TODO: re-run to check.")
@Test
public void testGetFusionModel() {
String project = "streaming_test";
diff --git a/src/query-service/src/test/java/org/apache/kylin/rest/service/QueryServiceTest.java b/src/query-service/src/test/java/org/apache/kylin/rest/service/QueryServiceTest.java
index fec084ee69..49e8eac47c 100644
--- a/src/query-service/src/test/java/org/apache/kylin/rest/service/QueryServiceTest.java
+++ b/src/query-service/src/test/java/org/apache/kylin/rest/service/QueryServiceTest.java
@@ -99,6 +99,7 @@ import org.apache.kylin.metadata.querymeta.TableMeta;
import org.apache.kylin.metadata.querymeta.TableMetaWithType;
import org.apache.kylin.metadata.realization.IRealization;
import org.apache.kylin.metadata.realization.RealizationStatusEnum;
+import org.apache.kylin.metadata.user.ManagedUser;
import org.apache.kylin.query.blacklist.SQLBlacklistItem;
import org.apache.kylin.query.blacklist.SQLBlacklistManager;
import org.apache.kylin.query.engine.PrepareSqlStateParam;
@@ -108,6 +109,7 @@ import org.apache.kylin.query.engine.data.QueryResult;
import org.apache.kylin.query.relnode.OLAPContext;
import org.apache.kylin.query.util.DateNumberFilterTransformer;
import org.apache.kylin.query.util.QueryParams;
+import org.apache.kylin.query.util.QueryUtil;
import org.apache.kylin.query.util.RawSqlParser;
import org.apache.kylin.rest.cluster.ClusterManager;
import org.apache.kylin.rest.cluster.DefaultClusterManager;
@@ -155,8 +157,6 @@ import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
-import org.apache.kylin.metadata.user.ManagedUser;
-import org.apache.kylin.query.util.KapQueryUtil;
import lombok.val;
/**
@@ -268,10 +268,10 @@ public class QueryServiceTest extends NLocalFileMetadataTestCase {
sqlRequest.setProject(project);
sqlRequest.setForcedToPushDown(true);
- QueryParams queryParams = new QueryParams(KapQueryUtil.getKylinConfig(sqlRequest.getProject()),
+ QueryParams queryParams = new QueryParams(NProjectManager.getProjectConfig(sqlRequest.getProject()),
sqlRequest.getSql(), sqlRequest.getProject(), sqlRequest.getLimit(), sqlRequest.getOffset(),
queryExec.getDefaultSchemaName(), true);
- String correctedSql = KapQueryUtil.massageSql(queryParams);
+ String correctedSql = QueryUtil.massageSql(queryParams);
Mockito.when(queryExec.executeQuery(correctedSql))
.thenThrow(new RuntimeException("shouldn't execute executeQuery"));
@@ -303,10 +303,10 @@ public class QueryServiceTest extends NLocalFileMetadataTestCase {
sqlRequest.setForcedToPushDown(false);
sqlRequest.setForcedToTieredStorage(1);
- QueryParams queryParams = new QueryParams(KapQueryUtil.getKylinConfig(sqlRequest.getProject()),
+ QueryParams queryParams = new QueryParams(NProjectManager.getProjectConfig(sqlRequest.getProject()),
sqlRequest.getSql(), sqlRequest.getProject(), sqlRequest.getLimit(), sqlRequest.getOffset(),
queryExec.getDefaultSchemaName(), true);
- String correctedSql = KapQueryUtil.massageSql(queryParams);
+ QueryUtil.massageSql(queryParams);
overwriteSystemProp("kylin.query.pushdown-enabled", "false");
Mockito.doThrow(new SQLException(new SQLException(QueryContext.ROUTE_USE_FORCEDTOTIEREDSTORAGE)))
@@ -326,10 +326,10 @@ public class QueryServiceTest extends NLocalFileMetadataTestCase {
sqlRequest.setProject(project);
sqlRequest.setForcedToTieredStorage(1);
- QueryParams queryParams = new QueryParams(KapQueryUtil.getKylinConfig(sqlRequest.getProject()),
+ QueryParams queryParams = new QueryParams(NProjectManager.getProjectConfig(sqlRequest.getProject()),
sqlRequest.getSql(), sqlRequest.getProject(), sqlRequest.getLimit(), sqlRequest.getOffset(),
queryExec.getDefaultSchemaName(), true);
- String correctedSql = KapQueryUtil.massageSql(queryParams);
+ String correctedSql = QueryUtil.massageSql(queryParams);
overwriteSystemProp("kylin.query.pushdown-enabled", "false");
Mockito.doThrow(new SQLException(new SQLException("No model found for OLAPContex")))
@@ -349,10 +349,10 @@ public class QueryServiceTest extends NLocalFileMetadataTestCase {
sqlRequest.setProject(project);
sqlRequest.setForcedToIndex(true);
- QueryParams queryParams = new QueryParams(KapQueryUtil.getKylinConfig(sqlRequest.getProject()),
+ QueryParams queryParams = new QueryParams(NProjectManager.getProjectConfig(sqlRequest.getProject()),
sqlRequest.getSql(), sqlRequest.getProject(), sqlRequest.getLimit(), sqlRequest.getOffset(),
queryExec.getDefaultSchemaName(), true);
- String correctedSql = KapQueryUtil.massageSql(queryParams);
+ String correctedSql = QueryUtil.massageSql(queryParams);
Mockito.when(queryExec.executeQuery(correctedSql))
.thenThrow(new RuntimeException("shouldnt execute queryexec"));
@@ -2409,11 +2409,11 @@ public class QueryServiceTest extends NLocalFileMetadataTestCase {
sqlRequest.setProject(project);
sqlRequest.setForcedToTieredStorage(0);
- QueryParams queryParams = new QueryParams(KapQueryUtil.getKylinConfig(sqlRequest.getProject()),
+ QueryParams queryParams = new QueryParams(NProjectManager.getProjectConfig(sqlRequest.getProject()),
sqlRequest.getSql(), sqlRequest.getProject(), sqlRequest.getLimit(), sqlRequest.getOffset(),
queryExec.getDefaultSchemaName(), true);
queryParams.setForcedToTieredStorage(ForceToTieredStorage.CH_FAIL_TO_DFS);
- String correctedSql = KapQueryUtil.massageSql(queryParams);
+ String correctedSql = QueryUtil.massageSql(queryParams);
Mockito.when(queryExec.executeQuery(correctedSql)).thenReturn(new QueryResult());
Mockito.doReturn(new QueryResult()).when(queryService.queryRoutingEngine).execute(Mockito.any(), Mockito.any());
@@ -2432,10 +2432,10 @@ public class QueryServiceTest extends NLocalFileMetadataTestCase {
sqlRequest.setProject(project);
sqlRequest.setForcedToTieredStorage(1);
- QueryParams queryParams = new QueryParams(KapQueryUtil.getKylinConfig(sqlRequest.getProject()),
+ QueryParams queryParams = new QueryParams(NProjectManager.getProjectConfig(sqlRequest.getProject()),
sqlRequest.getSql(), sqlRequest.getProject(), sqlRequest.getLimit(), sqlRequest.getOffset(),
queryExec.getDefaultSchemaName(), true);
- String correctedSql = KapQueryUtil.massageSql(queryParams);
+ String correctedSql = QueryUtil.massageSql(queryParams);
Throwable cause = new SQLException(QueryContext.ROUTE_USE_FORCEDTOTIEREDSTORAGE);
Mockito.doThrow(
@@ -2461,10 +2461,10 @@ public class QueryServiceTest extends NLocalFileMetadataTestCase {
sqlRequest.setForcedToIndex(true);
sqlRequest.setForcedToPushDown(false);
- QueryParams queryParams = new QueryParams(KapQueryUtil.getKylinConfig(sqlRequest.getProject()),
+ QueryParams queryParams = new QueryParams(NProjectManager.getProjectConfig(sqlRequest.getProject()),
sqlRequest.getSql(), sqlRequest.getProject(), sqlRequest.getLimit(), sqlRequest.getOffset(),
queryExec.getDefaultSchemaName(), true);
- String correctedSql = KapQueryUtil.massageSql(queryParams);
+ String correctedSql = QueryUtil.massageSql(queryParams);
Throwable cause = new KylinException(QueryErrorCode.FORCED_TO_TIEREDSTORAGE_AND_FORCE_TO_INDEX,
MsgPicker.getMsg().getForcedToTieredstorageAndForceToIndex());
@@ -2489,10 +2489,10 @@ public class QueryServiceTest extends NLocalFileMetadataTestCase {
sqlRequest.setProject(project);
sqlRequest.setForcedToTieredStorage(2);
- QueryParams queryParams = new QueryParams(KapQueryUtil.getKylinConfig(sqlRequest.getProject()),
+ QueryParams queryParams = new QueryParams(NProjectManager.getProjectConfig(sqlRequest.getProject()),
sqlRequest.getSql(), sqlRequest.getProject(), sqlRequest.getLimit(), sqlRequest.getOffset(),
"queryExec.getDefaultSchemaName()", true);
- String correctedSql = KapQueryUtil.massageSql(queryParams);
+ String correctedSql = QueryUtil.massageSql(queryParams);
Throwable cause = new KylinException(QueryErrorCode.FORCED_TO_TIEREDSTORAGE_RETURN_ERROR,
MsgPicker.getMsg().getForcedToTieredstorageReturnError());
diff --git a/src/query/src/main/java/io/kyligence/kap/query/optrule/AbstractAggCaseWhenFunctionRule.java b/src/query/src/main/java/io/kyligence/kap/query/optrule/AbstractAggCaseWhenFunctionRule.java
index 8c7cbae8e8..6a015f6ada 100644
--- a/src/query/src/main/java/io/kyligence/kap/query/optrule/AbstractAggCaseWhenFunctionRule.java
+++ b/src/query/src/main/java/io/kyligence/kap/query/optrule/AbstractAggCaseWhenFunctionRule.java
@@ -18,8 +18,8 @@
package io.kyligence.kap.query.optrule;
-import static org.apache.kylin.query.util.KapQueryUtil.isCast;
-import static org.apache.kylin.query.util.KapQueryUtil.isNotNullLiteral;
+import static org.apache.kylin.query.util.QueryUtil.isCast;
+import static org.apache.kylin.query.util.QueryUtil.isNotNullLiteral;
import java.math.BigDecimal;
import java.util.ArrayList;
diff --git a/src/query/src/main/java/io/kyligence/kap/query/optrule/CountDistinctCaseWhenFunctionRule.java b/src/query/src/main/java/io/kyligence/kap/query/optrule/CountDistinctCaseWhenFunctionRule.java
index c9cd452cb5..ddf0e6d303 100644
--- a/src/query/src/main/java/io/kyligence/kap/query/optrule/CountDistinctCaseWhenFunctionRule.java
+++ b/src/query/src/main/java/io/kyligence/kap/query/optrule/CountDistinctCaseWhenFunctionRule.java
@@ -18,9 +18,9 @@
package io.kyligence.kap.query.optrule;
-import static org.apache.kylin.query.util.KapQueryUtil.isCast;
-import static org.apache.kylin.query.util.KapQueryUtil.isNullLiteral;
-import static org.apache.kylin.query.util.KapQueryUtil.isPlainTableColumn;
+import static org.apache.kylin.query.util.QueryUtil.isCast;
+import static org.apache.kylin.query.util.QueryUtil.isNullLiteral;
+import static org.apache.kylin.query.util.QueryUtil.isPlainTableColumn;
import java.util.ArrayList;
import java.util.List;
diff --git a/src/query/src/main/java/io/kyligence/kap/query/optrule/KapAggFilterTransposeRule.java b/src/query/src/main/java/io/kyligence/kap/query/optrule/KapAggFilterTransposeRule.java
index 39fe5033e9..3567d52671 100644
--- a/src/query/src/main/java/io/kyligence/kap/query/optrule/KapAggFilterTransposeRule.java
+++ b/src/query/src/main/java/io/kyligence/kap/query/optrule/KapAggFilterTransposeRule.java
@@ -40,7 +40,7 @@ import org.apache.calcite.util.mapping.Mappings;
import org.apache.kylin.query.relnode.KapAggregateRel;
import org.apache.kylin.query.relnode.KapFilterRel;
import org.apache.kylin.query.relnode.KapJoinRel;
-import org.apache.kylin.query.util.KapQueryUtil;
+import org.apache.kylin.query.util.QueryUtil;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
@@ -68,7 +68,7 @@ public class KapAggFilterTransposeRule extends RelOptRule {
final KapJoinRel joinRel = call.rel(2);
//Only one agg child of join is accepted
- return KapQueryUtil.isJoinOnlyOneAggChild(joinRel);
+ return QueryUtil.isJoinOnlyOneAggChild(joinRel);
}
@Override
diff --git a/src/query/src/main/java/io/kyligence/kap/query/optrule/KapAggJoinTransposeRule.java b/src/query/src/main/java/io/kyligence/kap/query/optrule/KapAggJoinTransposeRule.java
index f368fcd380..e800a1166f 100644
--- a/src/query/src/main/java/io/kyligence/kap/query/optrule/KapAggJoinTransposeRule.java
+++ b/src/query/src/main/java/io/kyligence/kap/query/optrule/KapAggJoinTransposeRule.java
@@ -51,7 +51,7 @@ import org.apache.calcite.util.mapping.Mapping;
import org.apache.calcite.util.mapping.Mappings;
import org.apache.kylin.query.relnode.KapAggregateRel;
import org.apache.kylin.query.relnode.KapJoinRel;
-import org.apache.kylin.query.util.KapQueryUtil;
+import org.apache.kylin.query.util.QueryUtil;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
@@ -81,7 +81,7 @@ public class KapAggJoinTransposeRule extends RelOptRule {
final KapAggregateRel aggregate = call.rel(0);
final KapJoinRel joinRel = call.rel(1);
//Only one agg child of join is accepted
- return !aggregate.isContainCountDistinct() && KapQueryUtil.isJoinOnlyOneAggChild(joinRel);
+ return !aggregate.isContainCountDistinct() && QueryUtil.isJoinOnlyOneAggChild(joinRel);
}
@Override
diff --git a/src/query/src/main/java/io/kyligence/kap/query/optrule/KapAggProjectMergeRule.java b/src/query/src/main/java/io/kyligence/kap/query/optrule/KapAggProjectMergeRule.java
index e33a9242fb..6dad5cbb8e 100644
--- a/src/query/src/main/java/io/kyligence/kap/query/optrule/KapAggProjectMergeRule.java
+++ b/src/query/src/main/java/io/kyligence/kap/query/optrule/KapAggProjectMergeRule.java
@@ -39,7 +39,7 @@ import org.apache.kylin.query.relnode.KapAggregateRel;
import org.apache.kylin.query.relnode.KapFilterRel;
import org.apache.kylin.query.relnode.KapJoinRel;
import org.apache.kylin.query.relnode.KapProjectRel;
-import org.apache.kylin.query.util.KapQueryUtil;
+import org.apache.kylin.query.util.QueryUtil;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
@@ -77,7 +77,7 @@ public class KapAggProjectMergeRule extends RelOptRule {
}
//Only one agg child of join is accepted
- if (!KapQueryUtil.isJoinOnlyOneAggChild(joinRel)) {
+ if (!QueryUtil.isJoinOnlyOneAggChild(joinRel)) {
return false;
}
diff --git a/src/query/src/main/java/io/kyligence/kap/query/optrule/KapAggProjectTransposeRule.java b/src/query/src/main/java/io/kyligence/kap/query/optrule/KapAggProjectTransposeRule.java
index 24a031260c..f873bd08c2 100644
--- a/src/query/src/main/java/io/kyligence/kap/query/optrule/KapAggProjectTransposeRule.java
+++ b/src/query/src/main/java/io/kyligence/kap/query/optrule/KapAggProjectTransposeRule.java
@@ -48,7 +48,7 @@ import org.apache.kylin.query.relnode.KapAggregateRel;
import org.apache.kylin.query.relnode.KapFilterRel;
import org.apache.kylin.query.relnode.KapJoinRel;
import org.apache.kylin.query.relnode.KapProjectRel;
-import org.apache.kylin.query.util.KapQueryUtil;
+import org.apache.kylin.query.util.QueryUtil;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
@@ -89,7 +89,7 @@ public class KapAggProjectTransposeRule extends RelOptRule {
}
//Only one agg child of join is accepted
- if (!KapQueryUtil.isJoinOnlyOneAggChild(joinRel)) {
+ if (!QueryUtil.isJoinOnlyOneAggChild(joinRel)) {
return false;
}
diff --git a/src/query/src/main/java/io/kyligence/kap/query/optrule/KapCountDistinctJoinRule.java b/src/query/src/main/java/io/kyligence/kap/query/optrule/KapCountDistinctJoinRule.java
index 01d6630cc7..ce530aec1f 100644
--- a/src/query/src/main/java/io/kyligence/kap/query/optrule/KapCountDistinctJoinRule.java
+++ b/src/query/src/main/java/io/kyligence/kap/query/optrule/KapCountDistinctJoinRule.java
@@ -33,7 +33,7 @@ import org.apache.calcite.tools.RelBuilderFactory;
import org.apache.calcite.util.ImmutableBitSet;
import org.apache.kylin.query.relnode.KapAggregateRel;
import org.apache.kylin.query.relnode.KapJoinRel;
-import org.apache.kylin.query.util.KapQueryUtil;
+import org.apache.kylin.query.util.QueryUtil;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
@@ -82,7 +82,7 @@ public class KapCountDistinctJoinRule extends RelOptRule {
public boolean matches(RelOptRuleCall call) {
final KapAggregateRel aggregate = call.rel(0);
final KapJoinRel join = call.rel(1);
- return aggregate.isContainCountDistinct() && KapQueryUtil.isJoinOnlyOneAggChild(join);
+ return aggregate.isContainCountDistinct() && QueryUtil.isJoinOnlyOneAggChild(join);
}
@Override
diff --git a/src/query/src/main/java/io/kyligence/kap/query/optrule/KapSumCastTransposeRule.java b/src/query/src/main/java/io/kyligence/kap/query/optrule/KapSumCastTransposeRule.java
index 4293722cb2..f9608854d7 100644
--- a/src/query/src/main/java/io/kyligence/kap/query/optrule/KapSumCastTransposeRule.java
+++ b/src/query/src/main/java/io/kyligence/kap/query/optrule/KapSumCastTransposeRule.java
@@ -18,8 +18,8 @@
package io.kyligence.kap.query.optrule;
-import static org.apache.kylin.query.util.KapQueryUtil.containCast;
-import static org.apache.kylin.query.util.KapQueryUtil.isNotNullLiteral;
+import static org.apache.kylin.query.util.QueryUtil.containCast;
+import static org.apache.kylin.query.util.QueryUtil.isNotNullLiteral;
import java.math.BigDecimal;
import java.util.List;
diff --git a/src/query/src/main/java/org/apache/kylin/query/engine/QueryRoutingEngine.java b/src/query/src/main/java/org/apache/kylin/query/engine/QueryRoutingEngine.java
index 5937a43650..fe1d092ee3 100644
--- a/src/query/src/main/java/org/apache/kylin/query/engine/QueryRoutingEngine.java
+++ b/src/query/src/main/java/org/apache/kylin/query/engine/QueryRoutingEngine.java
@@ -52,7 +52,6 @@ import org.apache.kylin.metadata.realization.NoStreamingRealizationFoundExceptio
import org.apache.kylin.query.engine.data.QueryResult;
import org.apache.kylin.query.mask.QueryResultMasks;
import org.apache.kylin.query.relnode.OLAPContext;
-import org.apache.kylin.query.util.KapQueryUtil;
import org.apache.kylin.query.util.PushDownUtil;
import org.apache.kylin.query.util.QueryParams;
import org.apache.kylin.query.util.QueryUtil;
@@ -92,7 +91,7 @@ public class QueryRoutingEngine {
queryParams.setSql(queryParams.getPrepareSql());
}
- String correctedSql = KapQueryUtil.massageSql(queryParams);
+ String correctedSql = QueryUtil.massageSql(queryParams);
//CAUTION: should not change sqlRequest content!
QueryContext.current().getMetrics().setCorrectedSql(correctedSql);
@@ -216,7 +215,7 @@ public class QueryRoutingEngine {
private boolean checkBigQueryPushDown(QueryParams queryParams) {
KylinConfig kylinConfig = NProjectManager.getInstance(KylinConfig.getInstanceFromEnv())
.getProject(queryParams.getProject()).getConfig();
- boolean isPush = KapQueryUtil.checkBigQueryPushDown(kylinConfig);
+ boolean isPush = QueryUtil.checkBigQueryPushDown(kylinConfig);
if (isPush) {
logger.info("Big query route to pushdown.");
}
@@ -257,7 +256,7 @@ public class QueryRoutingEngine {
sqlString = QueryUtil.addLimit(sqlString);
}
- String massagedSql = KapQueryUtil.normalMassageSql(KylinConfig.getInstanceFromEnv(), sqlString,
+ String massagedSql = QueryUtil.normalMassageSql(KylinConfig.getInstanceFromEnv(), sqlString,
queryParams.getLimit(), queryParams.getOffset());
if (isPrepareStatementWithParams(queryParams)) {
QueryContext.current().getMetrics().setCorrectedSql(massagedSql);
diff --git a/src/query/src/main/java/org/apache/kylin/query/util/QueryHelper.java b/src/query/src/main/java/org/apache/kylin/query/util/QueryHelper.java
index 7f457bc423..4f6449f9b6 100644
--- a/src/query/src/main/java/org/apache/kylin/query/util/QueryHelper.java
+++ b/src/query/src/main/java/org/apache/kylin/query/util/QueryHelper.java
@@ -18,7 +18,8 @@
package org.apache.kylin.query.util;
-import lombok.val;
+import java.sql.SQLException;
+
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.Unsafe;
import org.apache.kylin.metadata.project.NProjectManager;
@@ -28,8 +29,7 @@ import org.apache.spark.sql.Row;
import org.apache.spark.sql.SparderEnv;
import org.apache.spark.sql.SparkSession;
-import java.sql.SQLException;
-
+import lombok.val;
public class QueryHelper {
@@ -42,11 +42,12 @@ public class QueryHelper {
public static Dataset<Row> singleQuery(String sql, String project) throws SQLException {
val prevRunLocalConf = Unsafe.setProperty(RUN_CONSTANT_QUERY_LOCALLY, "FALSE");
try {
- val projectKylinConfig = NProjectManager.getInstance(KylinConfig.getInstanceFromEnv()).getProject(project).getConfig();
+ val projectKylinConfig = NProjectManager.getInstance(KylinConfig.getInstanceFromEnv()).getProject(project)
+ .getConfig();
val queryExec = new QueryExec(project, projectKylinConfig);
- val queryParams = new QueryParams(KapQueryUtil.getKylinConfig(project),
- sql, project, 0, 0, queryExec.getDefaultSchemaName(), true);
- val convertedSql = KapQueryUtil.massageSql(queryParams);
+ val queryParams = new QueryParams(NProjectManager.getProjectConfig(project), sql, project, 0, 0,
+ queryExec.getDefaultSchemaName(), true);
+ val convertedSql = QueryUtil.massageSql(queryParams);
queryExec.executeQuery(convertedSql);
} finally {
if (prevRunLocalConf == null) {
diff --git a/src/query/src/main/java/org/apache/kylin/query/util/QueryUtil.java b/src/query/src/main/java/org/apache/kylin/query/util/QueryUtil.java
deleted file mode 100644
index 02fd6b8087..0000000000
--- a/src/query/src/main/java/org/apache/kylin/query/util/QueryUtil.java
+++ /dev/null
@@ -1,171 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.query.util;
-
-import java.util.Collections;
-import java.util.List;
-import java.util.Locale;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.query.engine.QueryExec;
-import org.apache.kylin.query.security.AccessDeniedException;
-import org.apache.spark.sql.catalyst.analysis.NoSuchDatabaseException;
-import org.apache.spark.sql.catalyst.analysis.NoSuchTableException;
-
-import com.google.common.collect.Lists;
-
-import lombok.extern.slf4j.Slf4j;
-
-@Slf4j
-public class QueryUtil {
-
- private static final Pattern SELECT_PATTERN = Pattern.compile("^select", Pattern.CASE_INSENSITIVE);
- private static final Pattern LIMIT_PATTERN = Pattern.compile("(limit\\s+[0-9;]+)$", Pattern.CASE_INSENSITIVE);
- static List<KapQueryUtil.IQueryTransformer> tableDetectTransformers = Collections.emptyList();
-
- private QueryUtil() {
- throw new IllegalStateException("Utility class");
- }
-
- public static String normalizeForTableDetecting(String project, String sql) {
- KylinConfig kylinConfig = KapQueryUtil.getKylinConfig(project);
- String convertedSql = KapQueryUtil.normalMassageSql(kylinConfig, sql, 0, 0);
- String defaultSchema = "DEFAULT";
- try {
- QueryExec queryExec = new QueryExec(project, kylinConfig);
- defaultSchema = queryExec.getDefaultSchemaName();
- } catch (Exception e) {
- log.error("Get project default schema failed.", e);
- }
-
- String[] detectorTransformers = kylinConfig.getTableDetectorTransformers();
- List<KapQueryUtil.IQueryTransformer> transformerList = KapQueryUtil.initTransformers(false, detectorTransformers);
- tableDetectTransformers = Collections.unmodifiableList(transformerList);
- for (KapQueryUtil.IQueryTransformer t : tableDetectTransformers) {
- convertedSql = t.transform(convertedSql, project, defaultSchema);
- }
- return convertedSql;
- }
-
- public static String makeErrorMsgUserFriendly(Throwable e) {
- String msg = e.getMessage();
-
- // pick ParseException error message if possible
- Throwable cause = e;
- while (cause != null) {
- if (cause.getClass().getName().contains("ParseException") || cause instanceof NoSuchTableException
- || cause instanceof NoSuchDatabaseException || cause instanceof AccessDeniedException) {
- msg = cause.getMessage();
- break;
- }
-
- if (cause.getClass().getName().contains("ArithmeticException")) {
- msg = "ArithmeticException: " + cause.getMessage();
- break;
- }
-
- if (cause.getClass().getName().contains("NoStreamingRealizationFoundException")) {
- msg = "NoStreamingRealizationFoundException: " + cause.getMessage();
- break;
- }
- cause = cause.getCause();
- }
-
- return makeErrorMsgUserFriendly(msg);
- }
-
- public static String makeErrorMsgUserFriendly(String errorMsg) {
- try {
- errorMsg = errorMsg.trim();
-
- // move cause to be ahead of sql, calcite creates the message pattern below
- Pattern pattern = Pattern.compile("Error while executing SQL ([\\s\\S]*):(.*):(.*)");
- Matcher matcher = pattern.matcher(errorMsg);
- if (matcher.find()) {
- return matcher.group(2).trim() + ": " + matcher.group(3).trim() + "\nwhile executing SQL: "
- + matcher.group(1).trim();
- } else
- return errorMsg;
- } catch (Exception e) {
- return errorMsg;
- }
- }
-
- public static boolean isSelectStatement(String sql) {
- String sql1 = sql.toLowerCase(Locale.ROOT);
- sql1 = removeCommentInSql(sql1);
- sql1 = sql1.trim();
- while (sql1.startsWith("(")) {
- sql1 = sql1.substring(1).trim();
- }
- return sql1.startsWith("select") || (sql1.startsWith("with") && sql1.contains("select"))
- || (sql1.startsWith("explain") && sql1.contains("select"));
- }
-
- public static String removeCommentInSql(String sql) {
- // match two patterns, one is "-- comment", the other is "/* comment */"
- try {
- return new RawSqlParser(sql).parse().getStatementString();
- } catch (Exception ex) {
- log.error("Something unexpected while removing comments in the query, return original query", ex);
- return sql;
- }
- }
-
- public static List<String> splitBySemicolon(String s) {
- List<String> r = Lists.newArrayList();
- StringBuilder sb = new StringBuilder();
- boolean inQuota = false;
- for (int i = 0; i < s.length(); i++) {
- if (s.charAt(i) == '\'') {
- inQuota = !inQuota;
- }
- if (s.charAt(i) == ';' && !inQuota) {
- if (sb.length() != 0) {
- r.add(sb.toString());
- sb = new StringBuilder();
- }
- continue;
- }
- sb.append(s.charAt(i));
- }
- if (sb.length() != 0) {
- r.add(sb.toString());
- }
- return r;
- }
-
- public static String addLimit(String originString) {
- Matcher selectMatcher = SELECT_PATTERN.matcher(originString);
- Matcher limitMatcher = LIMIT_PATTERN.matcher(originString);
- String replacedString = originString;
-
- if (selectMatcher.find() && !limitMatcher.find()) {
- if (originString.endsWith(";")) {
- replacedString = originString.replaceAll(";+$", "");
- }
-
- replacedString = replacedString.concat(" limit 1");
- }
-
- return replacedString;
- }
-}
diff --git a/src/query/src/test/java/org/apache/kylin/query/util/KapQueryUtilTest.java b/src/query/src/test/java/org/apache/kylin/query/util/KapQueryUtilTest.java
deleted file mode 100644
index fcacacddfe..0000000000
--- a/src/query/src/test/java/org/apache/kylin/query/util/KapQueryUtilTest.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.query.util;
-
-import static org.junit.Assert.assertEquals;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.mockStatic;
-import static org.mockito.Mockito.when;
-
-import org.apache.kylin.common.KylinConfig;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.mockito.MockedStatic;
-import org.mockito.junit.MockitoJUnitRunner;
-
-@RunWith(MockitoJUnitRunner.class)
-public class KapQueryUtilTest {
-
- public static final String SQL = "select * from table1";
-
- @Test
- public void testMaxResultRowsEnabled() {
- try (MockedStatic<KylinConfig> kylinConfigMockedStatic = mockStatic(KylinConfig.class)) {
- KylinConfig kylinConfig = mock(KylinConfig.class);
- kylinConfigMockedStatic.when(KylinConfig::getInstanceFromEnv).thenReturn(kylinConfig);
- when(kylinConfig.getMaxResultRows()).thenReturn(15);
- when(kylinConfig.getForceLimit()).thenReturn(14);
- String result = KapQueryUtil.normalMassageSql(kylinConfig, SQL, 16, 0);
- assertEquals("select * from table1" + "\n" + "LIMIT 15", result);
- }
- }
-
- @Test
- public void testCompareMaxResultRowsAndLimit() {
- try (MockedStatic<KylinConfig> kylinConfigMockedStatic = mockStatic(KylinConfig.class)) {
- KylinConfig kylinConfig = mock(KylinConfig.class);
- kylinConfigMockedStatic.when(KylinConfig::getInstanceFromEnv).thenReturn(kylinConfig);
- when(kylinConfig.getMaxResultRows()).thenReturn(15);
- when(kylinConfig.getForceLimit()).thenReturn(14);
- String result = KapQueryUtil.normalMassageSql(kylinConfig, SQL, 13, 0);
- assertEquals("select * from table1" + "\n" + "LIMIT 13", result);
- }
- }
-}
diff --git a/src/query/src/test/java/org/apache/kylin/query/util/QueryUtilTest.java b/src/query/src/test/java/org/apache/kylin/query/util/QueryUtilTest.java
index 684ed8ecba..1496a92d72 100644
--- a/src/query/src/test/java/org/apache/kylin/query/util/QueryUtilTest.java
+++ b/src/query/src/test/java/org/apache/kylin/query/util/QueryUtilTest.java
@@ -18,6 +18,11 @@
package org.apache.kylin.query.util;
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.mockStatic;
+import static org.mockito.Mockito.when;
+
import java.sql.SQLException;
import java.util.Collections;
import java.util.Properties;
@@ -33,13 +38,14 @@ import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
+import org.mockito.MockedStatic;
public class QueryUtilTest extends NLocalFileMetadataTestCase {
@Before
public void setUp() throws Exception {
- KapQueryUtil.queryTransformers = Collections.emptyList();
- KapQueryUtil.pushDownConverters = Collections.emptyList();
+ QueryUtil.queryTransformers = Collections.emptyList();
+ QueryUtil.pushDownConverters = Collections.emptyList();
this.createTestMetadata();
}
@@ -48,6 +54,32 @@ public class QueryUtilTest extends NLocalFileMetadataTestCase {
this.cleanupTestMetadata();
}
+ public static final String SQL = "select * from table1";
+
+ @Test
+ public void testMaxResultRowsEnabled() {
+ try (MockedStatic<KylinConfig> kylinConfigMockedStatic = mockStatic(KylinConfig.class)) {
+ KylinConfig kylinConfig = mock(KylinConfig.class);
+ kylinConfigMockedStatic.when(KylinConfig::getInstanceFromEnv).thenReturn(kylinConfig);
+ when(kylinConfig.getMaxResultRows()).thenReturn(15);
+ when(kylinConfig.getForceLimit()).thenReturn(14);
+ String result = QueryUtil.normalMassageSql(kylinConfig, SQL, 16, 0);
+ assertEquals("select * from table1" + "\n" + "LIMIT 15", result);
+ }
+ }
+
+ @Test
+ public void testCompareMaxResultRowsAndLimit() {
+ try (MockedStatic<KylinConfig> kylinConfigMockedStatic = mockStatic(KylinConfig.class)) {
+ KylinConfig kylinConfig = mock(KylinConfig.class);
+ kylinConfigMockedStatic.when(KylinConfig::getInstanceFromEnv).thenReturn(kylinConfig);
+ when(kylinConfig.getMaxResultRows()).thenReturn(15);
+ when(kylinConfig.getForceLimit()).thenReturn(14);
+ String result = QueryUtil.normalMassageSql(kylinConfig, SQL, 13, 0);
+ assertEquals("select * from table1" + "\n" + "LIMIT 13", result);
+ }
+ }
+
@Test
public void testMassageSql() {
KylinConfig config = KylinConfig.createKylinConfig(new Properties());
@@ -56,12 +88,12 @@ public class QueryUtilTest extends NLocalFileMetadataTestCase {
String sql = "SELECT * FROM TABLE";
QueryParams queryParams1 = new QueryParams(config, sql, "", 100, 20, "", true);
- String newSql = KapQueryUtil.massageSql(queryParams1);
+ String newSql = QueryUtil.massageSql(queryParams1);
Assert.assertEquals("SELECT * FROM TABLE\nLIMIT 100\nOFFSET 20", newSql);
String sql2 = "SELECT SUM({fn convert(0, INT)}) from TABLE";
QueryParams queryParams2 = new QueryParams(config, sql2, "", 0, 0, "", true);
- String newSql2 = KapQueryUtil.massageSql(queryParams2);
+ String newSql2 = QueryUtil.massageSql(queryParams2);
Assert.assertEquals("SELECT SUM({fn convert(0, INT)}) from TABLE", newSql2);
}
}
@@ -75,22 +107,22 @@ public class QueryUtilTest extends NLocalFileMetadataTestCase {
config.setProperty("kylin.query.transformers", "org.apache.kylin.query.util.ConvertToComputedColumn");
QueryParams queryParams1 = new QueryParams(config, "SELECT price * item_count FROM test_kylin_fact",
"default", 0, 0, "DEFAULT", true);
- String newSql1 = KapQueryUtil.massageSql(queryParams1);
+ String newSql1 = QueryUtil.massageSql(queryParams1);
Assert.assertEquals("SELECT TEST_KYLIN_FACT.DEAL_AMOUNT FROM test_kylin_fact", newSql1);
QueryParams queryParams2 = new QueryParams(config,
"SELECT price * item_count,DEAL_AMOUNT FROM test_kylin_fact", "default", 0, 0, "DEFAULT", true);
- newSql1 = KapQueryUtil.massageSql(queryParams2);
+ newSql1 = QueryUtil.massageSql(queryParams2);
Assert.assertEquals("SELECT TEST_KYLIN_FACT.DEAL_AMOUNT,DEAL_AMOUNT FROM test_kylin_fact", newSql1);
// disable ConvertToComputedColumn
config.setProperty("kylin.query.transformers", "");
QueryParams queryParams3 = new QueryParams(config, "SELECT price * item_count FROM test_kylin_fact",
"default", 0, 0, "DEFAULT", true);
- String newSql2 = KapQueryUtil.massageSql(queryParams3);
+ String newSql2 = QueryUtil.massageSql(queryParams3);
Assert.assertEquals("SELECT price * item_count FROM test_kylin_fact", newSql2);
QueryParams queryParams4 = new QueryParams(config,
"SELECT price * item_count,DEAL_AMOUNT FROM test_kylin_fact", "default", 0, 0, "DEFAULT", false);
- newSql2 = KapQueryUtil.massageSql(queryParams4);
+ newSql2 = QueryUtil.massageSql(queryParams4);
Assert.assertEquals("SELECT price * item_count,DEAL_AMOUNT FROM test_kylin_fact", newSql2);
}
}
@@ -120,7 +152,7 @@ public class QueryUtilTest extends NLocalFileMetadataTestCase {
String expected = "select TEST_KYLIN_FACT.TMP_CC from test_kylin_fact left join TEST_CATEGORY_GROUPINGS "
+ "on TEST_KYLIN_FACT.LEAF_CATEG_ID = TEST_CATEGORY_GROUPINGS.LEAF_CATEG_ID and TEST_KYLIN_FACT.LSTG_SITE_ID = TEST_CATEGORY_GROUPINGS.SITE_ID";
QueryParams queryParams = new QueryParams(config, sql, "default", 0, 0, "DEFAULT", true);
- String result = KapQueryUtil.massageSql(queryParams);
+ String result = QueryUtil.massageSql(queryParams);
Assert.assertEquals(expected, result);
// join condition is colName = colName
@@ -129,7 +161,7 @@ public class QueryUtilTest extends NLocalFileMetadataTestCase {
String expected2 = "select TEST_KYLIN_FACT.TMP_CC from test_kylin_fact left join TEST_CATEGORY_GROUPINGS "
+ "on TEST_KYLIN_FACT.LEAF_CATEG_ID = TEST_CATEGORY_GROUPINGS.LEAF_CATEG_ID and LSTG_SITE_ID = SITE_ID";
QueryParams queryParams2 = new QueryParams(config, sql2, "default", 0, 0, "DEFAULT", true);
- String result2 = KapQueryUtil.massageSql(queryParams2);
+ String result2 = QueryUtil.massageSql(queryParams2);
Assert.assertEquals(expected2, result2);
// join condition is colName = colName
@@ -140,7 +172,7 @@ public class QueryUtilTest extends NLocalFileMetadataTestCase {
+ "ON \"DEFAULT\".TEST_KYLIN_FACT.LEAF_CATEG_ID = \"DEFAULT\".TEST_CATEGORY_GROUPINGS.LEAF_CATEG_ID "
+ "AND \"DEFAULT\".TEST_KYLIN_FACT.LSTG_SITE_ID = \"DEFAULT\".TEST_CATEGORY_GROUPINGS.SITE_ID";
QueryParams queryParams3 = new QueryParams(config, sql3, "default", 0, 0, "DEFAULT", true);
- String result3 = KapQueryUtil.massageSql(queryParams3);
+ String result3 = QueryUtil.massageSql(queryParams3);
Assert.assertEquals(expected3, result3);
}
@@ -156,7 +188,7 @@ public class QueryUtilTest extends NLocalFileMetadataTestCase {
QueryParams queryParams = new QueryParams("", sql, "default", false);
queryParams.setKylinConfig(config);
- String massagedSql = KapQueryUtil.massagePushDownSql(queryParams);
+ String massagedSql = QueryUtil.massagePushDownSql(queryParams);
String expectedSql = "SELECT `Z_PROVDASH_UM_ED`.`GENDER` AS `GENDER`,\n"
+ "SUM(CAST(0 AS BIGINT)) AS `sum_Calculation_336925569152049156_ok`\n"
+ "FROM `POPHEALTH_ANALYTICS`.`Z_PROVDASH_UM_ED` `Z_PROVDASH_UM_ED`";
@@ -170,27 +202,27 @@ public class QueryUtilTest extends NLocalFileMetadataTestCase {
try (SetAndUnsetThreadLocalConfig autoUnset = KylinConfig.setAndUnsetThreadLocalConfig(config)) {
config.setProperty("kylin.query.transformers", DefaultQueryTransformer.class.getCanonicalName());
- Assert.assertEquals(0, KapQueryUtil.queryTransformers.size());
- KapQueryUtil.initQueryTransformersIfNeeded(config, true);
- Assert.assertEquals(1, KapQueryUtil.queryTransformers.size());
- Assert.assertTrue(KapQueryUtil.queryTransformers.get(0) instanceof DefaultQueryTransformer);
+ Assert.assertEquals(0, QueryUtil.queryTransformers.size());
+ QueryUtil.initQueryTransformersIfNeeded(config, true);
+ Assert.assertEquals(1, QueryUtil.queryTransformers.size());
+ Assert.assertTrue(QueryUtil.queryTransformers.get(0) instanceof DefaultQueryTransformer);
config.setProperty("kylin.query.transformers", KeywordDefaultDirtyHack.class.getCanonicalName());
- KapQueryUtil.initQueryTransformersIfNeeded(config, true);
- Assert.assertEquals(1, KapQueryUtil.queryTransformers.size());
- Assert.assertTrue(KapQueryUtil.queryTransformers.get(0) instanceof KeywordDefaultDirtyHack);
+ QueryUtil.initQueryTransformersIfNeeded(config, true);
+ Assert.assertEquals(1, QueryUtil.queryTransformers.size());
+ Assert.assertTrue(QueryUtil.queryTransformers.get(0) instanceof KeywordDefaultDirtyHack);
- KapQueryUtil.initQueryTransformersIfNeeded(config, false);
- Assert.assertEquals(1, KapQueryUtil.queryTransformers.size());
- Assert.assertTrue(KapQueryUtil.queryTransformers.get(0) instanceof KeywordDefaultDirtyHack);
+ QueryUtil.initQueryTransformersIfNeeded(config, false);
+ Assert.assertEquals(1, QueryUtil.queryTransformers.size());
+ Assert.assertTrue(QueryUtil.queryTransformers.get(0) instanceof KeywordDefaultDirtyHack);
config.setProperty("kylin.query.transformers", DefaultQueryTransformer.class.getCanonicalName() + ","
+ ConvertToComputedColumn.class.getCanonicalName());
- KapQueryUtil.initQueryTransformersIfNeeded(config, true);
- Assert.assertEquals(2, KapQueryUtil.queryTransformers.size());
- KapQueryUtil.initQueryTransformersIfNeeded(config, false);
- Assert.assertEquals(1, KapQueryUtil.queryTransformers.size());
- Assert.assertTrue(KapQueryUtil.queryTransformers.get(0) instanceof DefaultQueryTransformer);
+ QueryUtil.initQueryTransformersIfNeeded(config, true);
+ Assert.assertEquals(2, QueryUtil.queryTransformers.size());
+ QueryUtil.initQueryTransformersIfNeeded(config, false);
+ Assert.assertEquals(1, QueryUtil.queryTransformers.size());
+ Assert.assertTrue(QueryUtil.queryTransformers.get(0) instanceof DefaultQueryTransformer);
}
}
@@ -217,6 +249,15 @@ public class QueryUtilTest extends NLocalFileMetadataTestCase {
Assert.assertEquals(accessDeniedMsg, errorMessage);
}
+ @Test
+ public void testErrorMsg() {
+ String errorMsg = "Error while executing SQL \"select lkp.clsfd_ga_prfl_id, ga.sum_dt, sum(ga.bounces) as bounces, sum(ga.exits) as exits, sum(ga.entrances) as entrances, sum(ga.pageviews) as pageviews, count(distinct ga.GA_VSTR_ID, ga.GA_VST_ID) as visits, count(distinct ga.GA_VSTR_ID) as uniqVistors from CLSFD_GA_PGTYPE_CATEG_LOC ga left join clsfd_ga_prfl_lkp lkp on ga.SRC_GA_PRFL_ID = lkp.SRC_GA_PRFL_ID group by lkp.clsfd_ga_prfl_id,ga.sum_dt order by lkp.clsfd_ga_prfl_id,ga. [...]
+ Assert.assertEquals(
+ "From line 14, column 14 to line 14, column 29: Column 'CLSFD_GA_PRFL_ID' not found in table 'LKP'\n"
+ + "while executing SQL: \"select lkp.clsfd_ga_prfl_id, ga.sum_dt, sum(ga.bounces) as bounces, sum(ga.exits) as exits, sum(ga.entrances) as entrances, sum(ga.pageviews) as pageviews, count(distinct ga.GA_VSTR_ID, ga.GA_VST_ID) as visits, count(distinct ga.GA_VSTR_ID) as uniqVistors from CLSFD_GA_PGTYPE_CATEG_LOC ga left join clsfd_ga_prfl_lkp lkp on ga.SRC_GA_PRFL_ID = lkp.SRC_GA_PRFL_ID group by lkp.clsfd_ga_prfl_id,ga.sum_dt order by lkp.clsfd_ga_prfl_id,ga.sum_d [...]
+ QueryUtil.makeErrorMsgUserFriendly(errorMsg));
+ }
+
@Test
public void testJudgeSelectStatementStartsWithParentheses() {
String sql = "(((SELECT COUNT(DISTINCT \"LO_SUPPKEY\"), \"LO_SUPPKEY\", \"LO_ORDERKEY\", \"LO_ORDERDATE\", \"LO_PARTKEY\", \"LO_REVENUE\" "
@@ -320,7 +361,7 @@ public class QueryUtilTest extends NLocalFileMetadataTestCase {
String sql = "select sum(cast(CC1 as double)) from test_kylin_fact";
String expected = "select SUM(\"TEST_KYLIN_FACT\".\"PRICE\" + 1) from test_kylin_fact";
QueryParams queryParams = new QueryParams(config, sql, "default", 0, 0, "DEFAULT", true);
- Assert.assertEquals(expected, KapQueryUtil.massageSqlAndExpandCC(queryParams));
+ Assert.assertEquals(expected, QueryUtil.massageSqlAndExpandCC(queryParams));
}
@Test
@@ -330,7 +371,33 @@ public class QueryUtilTest extends NLocalFileMetadataTestCase {
+ " on s.BUYER_ID = a.ACCOUNT_ID inner join TEST_COUNTRY c on c.COUNTRY = a.ACCOUNT_COUNTRY\n"
+ " limit 10000)t\n";
String replacedString = QueryUtil.addLimit(originString);
- Assert.assertEquals(originString.concat(" limit 1"), replacedString);
+ Assert.assertEquals(originString.trim().concat(" limit 1"), replacedString);
+ }
+
+ @Test
+ public void testAddLimitWithSemicolon() {
+ String origin = "select a from t;;;;\n\t;;;\n;";
+ Assert.assertEquals("select a from t limit 1", QueryUtil.addLimit(origin));
+
+ origin = "select a from t limit 10;";
+ Assert.assertEquals(origin, QueryUtil.addLimit(origin));
+
+ origin = "select a from t limit 10; ;\t;\n;";
+ Assert.assertEquals(origin, QueryUtil.addLimit(origin));
+ }
+
+ @Test
+ public void testAddLimitWithEmpty() {
+ String origin = " ";
+ Assert.assertEquals(origin, QueryUtil.addLimit(origin));
+
+ Assert.assertNull(QueryUtil.addLimit(null));
+ }
+
+ @Test
+ public void testAddLimitNonSelect() {
+ String origin = "aaa";
+ Assert.assertEquals(origin, QueryUtil.addLimit(origin));
}
@Test
@@ -338,14 +405,14 @@ public class QueryUtilTest extends NLocalFileMetadataTestCase {
KylinConfig config = KylinConfig.getInstanceFromEnv();
String sql1 = "select EXTRACT(minute FROM lineorder.lo_orderdate) from lineorder inner join customer on lineorder.lo_custkey = customer.c_custkey";
QueryParams queryParams1 = new QueryParams(config, sql1, "cc_test", 0, 0, "ssb", true);
- String newSql1 = KapQueryUtil.massageSql(queryParams1);
+ String newSql1 = QueryUtil.massageSql(queryParams1);
Assert.assertEquals(
"select LINEORDER.CC_EXTRACT from lineorder inner join customer on lineorder.lo_custkey = customer.c_custkey",
newSql1);
String sql2 = "select {fn convert(lineorder.lo_orderkey, double)} from lineorder inner join customer on lineorder.lo_custkey = customer.c_custkey";
QueryParams queryParams2 = new QueryParams(config, sql2, "cc_test", 0, 0, "ssb", true);
- String newSql2 = KapQueryUtil.massageSql(queryParams2);
+ String newSql2 = QueryUtil.massageSql(queryParams2);
Assert.assertEquals(
"select LINEORDER.CC_CAST_LO_ORDERKEY from lineorder inner join customer on lineorder.lo_custkey = customer.c_custkey",
newSql2);
@@ -357,7 +424,7 @@ public class QueryUtilTest extends NLocalFileMetadataTestCase {
KylinConfig config = KylinConfig.getInstanceFromEnv();
String sql1 = "select TRANS_ID as test_limit, ORDER_ID as test_offset from TEST_KYLIN_FACT group by TRANS_ID, ORDER_ID";
QueryParams queryParams1 = new QueryParams(config, sql1, "default", 5, 2, "DEFAULT", true);
- String newSql1 = KapQueryUtil.massageSql(queryParams1);
+ String newSql1 = QueryUtil.massageSql(queryParams1);
Assert.assertEquals(
"select TRANS_ID as test_limit, ORDER_ID as test_offset from TEST_KYLIN_FACT group by TRANS_ID, ORDER_ID\n"
+ "LIMIT 5\n" + "OFFSET 2",
@@ -365,7 +432,7 @@ public class QueryUtilTest extends NLocalFileMetadataTestCase {
String sql2 = "select TRANS_ID as test_limit, ORDER_ID as test_offset from TEST_KYLIN_FACT group by TRANS_ID, ORDER_ID limit 10 offset 3";
QueryParams queryParams2 = new QueryParams(config, sql2, "cc_test", 5, 2, "ssb", true);
- String newSql2 = KapQueryUtil.massageSql(queryParams2);
+ String newSql2 = QueryUtil.massageSql(queryParams2);
Assert.assertEquals(
"select TRANS_ID as test_limit, ORDER_ID as test_offset from TEST_KYLIN_FACT group by TRANS_ID, ORDER_ID "
+ "limit 10 offset 3",
@@ -373,7 +440,7 @@ public class QueryUtilTest extends NLocalFileMetadataTestCase {
String sql3 = "(select TRANS_ID as test_limit, ORDER_ID as test_offset from TEST_KYLIN_FACT group by TRANS_ID, ORDER_ID)limit 10 offset 3";
QueryParams queryParams3 = new QueryParams(config, sql3, "cc_test", 5, 2, "ssb", true);
- String newSql3 = KapQueryUtil.massageSql(queryParams3);
+ String newSql3 = QueryUtil.massageSql(queryParams3);
Assert.assertEquals(
"(select TRANS_ID as test_limit, ORDER_ID as test_offset from TEST_KYLIN_FACT group by TRANS_ID, ORDER_ID)"
+ "limit 10 offset 3",
@@ -381,7 +448,7 @@ public class QueryUtilTest extends NLocalFileMetadataTestCase {
String sql4 = "select TRANS_ID as test_limit, ORDER_ID as \"limit\" from TEST_KYLIN_FACT group by TRANS_ID, ORDER_ID";
QueryParams queryParams4 = new QueryParams(config, sql4, "cc_test", 5, 2, "ssb", true);
- String newSql4 = KapQueryUtil.massageSql(queryParams4);
+ String newSql4 = QueryUtil.massageSql(queryParams4);
Assert.assertEquals(
"select TRANS_ID as test_limit, ORDER_ID as \"limit\" from TEST_KYLIN_FACT group by TRANS_ID, ORDER_ID\n"
+ "LIMIT 5\n" + "OFFSET 2",
@@ -389,12 +456,12 @@ public class QueryUtilTest extends NLocalFileMetadataTestCase {
String sql5 = "select '\"`OFFSET`\"'";
QueryParams queryParams5 = new QueryParams(config, sql5, "cc_test", 1, 4, "ssb", true);
- String newSql5 = KapQueryUtil.massageSql(queryParams5);
+ String newSql5 = QueryUtil.massageSql(queryParams5);
Assert.assertEquals("select '\"`OFFSET`\"'\n" + "LIMIT 1\n" + "OFFSET 4", newSql5);
String sql6 = "select TRANS_ID as \"offset\", \"limit\" as \"offset limit\" from TEST_KYLIN_FACT group by TRANS_ID, \"limit\"";
QueryParams queryParams6 = new QueryParams(config, sql6, "cc_test", 10, 5, "ssb", true);
- String newSql6 = KapQueryUtil.massageSql(queryParams6);
+ String newSql6 = QueryUtil.massageSql(queryParams6);
Assert.assertEquals(
"select TRANS_ID as \"offset\", \"limit\" as \"offset limit\" from TEST_KYLIN_FACT group by TRANS_ID, \"limit\"\n"
+ "LIMIT 10\n" + "OFFSET 5",
@@ -415,7 +482,7 @@ public class QueryUtilTest extends NLocalFileMetadataTestCase {
QueryParams queryParams = new QueryParams("", sql, "default", false);
queryParams.setKylinConfig(config);
- String massagedSql = KapQueryUtil.massagePushDownSql(queryParams);
+ String massagedSql = QueryUtil.massagePushDownSql(queryParams);
String expectedSql = "SELECT `Z_PROVDASH_UM_ED`.`GENDER` AS `GENDER`, "
+ "SUM(CAST(0 AS BIGINT)) AS `sum_Calculation_336925569152049156_ok`\n"
+ "FROM `POPHEALTH_ANALYTICS`.`Z_PROVDASH_UM_ED` AS `Z_PROVDASH_UM_ED`\n" + "LIMIT 1";
@@ -431,7 +498,7 @@ public class QueryUtilTest extends NLocalFileMetadataTestCase {
config.setProperty("kylin.query.big-query-pushdown", "true");
String sql1 = "select TRANS_ID as test_limit, ORDER_ID as test_offset from TEST_KYLIN_FACT group by TRANS_ID, ORDER_ID";
QueryParams queryParams1 = new QueryParams(config, sql1, "default", 0, 0, "DEFAULT", true);
- String newSql1 = KapQueryUtil.massageSql(queryParams1);
+ String newSql1 = QueryUtil.massageSql(queryParams1);
Assert.assertEquals(
"select TRANS_ID as test_limit, ORDER_ID as test_offset from TEST_KYLIN_FACT group by TRANS_ID, ORDER_ID\n"
+ "LIMIT 10",
@@ -443,31 +510,31 @@ public class QueryUtilTest extends NLocalFileMetadataTestCase {
KylinConfig config = KylinConfig.createKylinConfig(new Properties());
String sql1 = "select TRANS_ID as test_limit, ORDER_ID as test_offset from TEST_KYLIN_FACT group by TRANS_ID, ORDER_ID";
QueryParams queryParams1 = new QueryParams(config, sql1, "default", 0, 0, "DEFAULT", true);
- String newSql1 = KapQueryUtil.massageSql(queryParams1);
+ String newSql1 = QueryUtil.massageSql(queryParams1);
Assert.assertEquals(
"select TRANS_ID as test_limit, ORDER_ID as test_offset from TEST_KYLIN_FACT group by TRANS_ID, ORDER_ID",
newSql1);
String sql = "select TRANS_ID as test_limit, ORDER_ID as test_offset from TEST_KYLIN_FACT group by TRANS_ID, ORDER_ID";
QueryParams queryParams = new QueryParams(config, sql, "default", 0, 0, "DEFAULT", true);
- String targetSQL = KapQueryUtil.massageSql(queryParams);
+ String targetSQL = QueryUtil.massageSql(queryParams);
Assert.assertEquals(
"select TRANS_ID as test_limit, ORDER_ID as test_offset from TEST_KYLIN_FACT group by TRANS_ID, ORDER_ID",
targetSQL);
queryParams = new QueryParams(config, sql, "default", 1, 0, "DEFAULT", true);
- targetSQL = KapQueryUtil.massageSql(queryParams);
+ targetSQL = QueryUtil.massageSql(queryParams);
Assert.assertEquals(
"select TRANS_ID as test_limit, ORDER_ID as test_offset from TEST_KYLIN_FACT group by TRANS_ID, ORDER_ID\n"
+ "LIMIT 1",
targetSQL);
config.setProperty("kylin.query.max-result-rows", "2");
queryParams = new QueryParams(config, sql, "default", 0, 0, "DEFAULT", true);
- targetSQL = KapQueryUtil.massageSql(queryParams);
+ targetSQL = QueryUtil.massageSql(queryParams);
Assert.assertEquals(
"select TRANS_ID as test_limit, ORDER_ID as test_offset from TEST_KYLIN_FACT group by TRANS_ID, ORDER_ID\n"
+ "LIMIT 2",
targetSQL);
queryParams = new QueryParams(config, sql, "default", 1, 0, "DEFAULT", true);
- targetSQL = KapQueryUtil.massageSql(queryParams);
+ targetSQL = QueryUtil.massageSql(queryParams);
Assert.assertEquals(
"select TRANS_ID as test_limit, ORDER_ID as test_offset from TEST_KYLIN_FACT group by TRANS_ID, ORDER_ID\n"
+ "LIMIT 1",
@@ -475,40 +542,40 @@ public class QueryUtilTest extends NLocalFileMetadataTestCase {
config.setProperty("kylin.query.max-result-rows", "-1");
config.setProperty("kylin.query.force-limit", "3");
queryParams = new QueryParams(config, sql, "default", 0, 0, "DEFAULT", true);
- targetSQL = KapQueryUtil.massageSql(queryParams);
+ targetSQL = QueryUtil.massageSql(queryParams);
Assert.assertEquals(
"select TRANS_ID as test_limit, ORDER_ID as test_offset from TEST_KYLIN_FACT group by TRANS_ID, ORDER_ID",
targetSQL);
queryParams = new QueryParams(config, sql, "default", 1, 0, "DEFAULT", true);
- targetSQL = KapQueryUtil.massageSql(queryParams);
+ targetSQL = QueryUtil.massageSql(queryParams);
Assert.assertEquals(
"select TRANS_ID as test_limit, ORDER_ID as test_offset from TEST_KYLIN_FACT group by TRANS_ID, ORDER_ID\n"
+ "LIMIT 1",
targetSQL);
sql1 = "select * from table1";
queryParams = new QueryParams(config, sql1, "default", 0, 0, "DEFAULT", true);
- targetSQL = KapQueryUtil.massageSql(queryParams);
+ targetSQL = QueryUtil.massageSql(queryParams);
Assert.assertEquals("select * from table1" + "\n" + "LIMIT 3", targetSQL);
queryParams = new QueryParams(config, sql1, "default", 2, 0, "DEFAULT", true);
- targetSQL = KapQueryUtil.massageSql(queryParams);
+ targetSQL = QueryUtil.massageSql(queryParams);
Assert.assertEquals("select * from table1" + "\n" + "LIMIT 2", targetSQL);
sql1 = "select * from table1 limit 4";
queryParams = new QueryParams(config, sql1, "default", 0, 0, "DEFAULT", true);
- targetSQL = KapQueryUtil.massageSql(queryParams);
+ targetSQL = QueryUtil.massageSql(queryParams);
Assert.assertEquals("select * from table1 limit 4", targetSQL);
queryParams = new QueryParams(config, sql1, "default", 2, 0, "DEFAULT", true);
- targetSQL = KapQueryUtil.massageSql(queryParams);
+ targetSQL = QueryUtil.massageSql(queryParams);
Assert.assertEquals("select * from table1 limit 4", targetSQL);
config.setProperty("kylin.query.force-limit", "-1");
config.setProperty("kylin.query.share-state-switch-implement", "jdbc");
queryParams = new QueryParams(config, sql, "default", 0, 0, "DEFAULT", true);
- targetSQL = KapQueryUtil.massageSql(queryParams);
+ targetSQL = QueryUtil.massageSql(queryParams);
Assert.assertEquals(
"select TRANS_ID as test_limit, ORDER_ID as test_offset from TEST_KYLIN_FACT group by TRANS_ID, ORDER_ID",
targetSQL);
config.setProperty("kylin.query.big-query-pushdown", "true");
queryParams = new QueryParams(config, sql, "default", 0, 0, "DEFAULT", true);
- targetSQL = KapQueryUtil.massageSql(queryParams);
+ targetSQL = QueryUtil.massageSql(queryParams);
Assert.assertEquals(
"select TRANS_ID as test_limit, ORDER_ID as test_offset from TEST_KYLIN_FACT group by TRANS_ID, ORDER_ID",
targetSQL);
diff --git a/src/second-storage/clickhouse-it/src/test/java/io/kyligence/kap/secondstorage/tdvt/TDVTHiveTest.java b/src/second-storage/clickhouse-it/src/test/java/io/kyligence/kap/secondstorage/tdvt/TDVTHiveTest.java
index 8e8769589c..124c76b4fc 100644
--- a/src/second-storage/clickhouse-it/src/test/java/io/kyligence/kap/secondstorage/tdvt/TDVTHiveTest.java
+++ b/src/second-storage/clickhouse-it/src/test/java/io/kyligence/kap/secondstorage/tdvt/TDVTHiveTest.java
@@ -37,7 +37,9 @@ import org.apache.kylin.common.QueryContext;
import org.apache.kylin.common.util.Unsafe;
import org.apache.kylin.engine.spark.IndexDataConstructor;
import org.apache.kylin.metadata.model.NDataModelManager;
+import org.apache.kylin.metadata.project.NProjectManager;
import org.apache.kylin.query.util.QueryParams;
+import org.apache.kylin.query.util.QueryUtil;
import org.apache.kylin.util.ExecAndComp;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Row;
@@ -61,7 +63,6 @@ import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import io.kyligence.kap.newten.clickhouse.ClickHouseUtils;
-import org.apache.kylin.query.util.KapQueryUtil;
import io.kyligence.kap.secondstorage.SecondStorageUtil;
import io.kyligence.kap.secondstorage.test.ClickHouseClassRule;
import io.kyligence.kap.secondstorage.test.EnableClickHouseJob;
@@ -87,7 +88,7 @@ public class TDVTHiveTest {
@ClassRule
public static SharedSparkSession sharedSpark = new SharedSparkSession(
- ImmutableMap.of("spark.sql.extensions", "org.apache.kylin.query.SQLPushDownExtensions"));
+ ImmutableMap.of("spark.sql.extensions", "io.kyligence.kap.query.SQLPushDownExtensions"));
public static EnableTestUser enableTestUser = new EnableTestUser();
public static ClickHouseClassRule clickHouse = new ClickHouseClassRule(clickhouseNumber);
@@ -194,8 +195,8 @@ public class TDVTHiveTest {
private String runWithHive(String sqlStatement) {
QueryParams queryParams = new QueryParams(project, sqlStatement, "default", false);
- queryParams.setKylinConfig(KapQueryUtil.getKylinConfig(project));
- String afterConvert = KapQueryUtil.massagePushDownSql(queryParams);
+ queryParams.setKylinConfig(NProjectManager.getProjectConfig(project));
+ String afterConvert = QueryUtil.massagePushDownSql(queryParams);
// Table schema comes from csv and DATABASE.TABLE is not supported.
String sqlForSpark = ExecAndComp.removeDataBaseInSql(afterConvert);
Dataset<Row> plan = ExecAndComp.querySparkSql(sqlForSpark);
diff --git a/src/second-storage/core-ui/src/test/java/org/apache/kylin/rest/service/ModelServiceWithSecondStorageTest.java b/src/second-storage/core-ui/src/test/java/org/apache/kylin/rest/service/ModelServiceWithSecondStorageTest.java
index 79ec883d6f..3df761f6c2 100644
--- a/src/second-storage/core-ui/src/test/java/org/apache/kylin/rest/service/ModelServiceWithSecondStorageTest.java
+++ b/src/second-storage/core-ui/src/test/java/org/apache/kylin/rest/service/ModelServiceWithSecondStorageTest.java
@@ -38,7 +38,7 @@ import org.apache.kylin.metadata.project.EnhancedUnitOfWork;
import org.apache.kylin.metadata.project.NProjectManager;
import org.apache.kylin.metadata.query.QueryTimesResponse;
import org.apache.kylin.metadata.recommendation.candidate.JdbcRawRecStore;
-import org.apache.kylin.query.util.KapQueryUtil;
+import org.apache.kylin.query.util.QueryUtil;
import org.apache.kylin.rest.config.initialize.ModelBrokenListener;
import org.apache.kylin.rest.constant.Constant;
import org.apache.kylin.rest.request.ModelRequest;
@@ -141,7 +141,7 @@ public class ModelServiceWithSecondStorageTest extends NLocalFileMetadataTestCas
ReflectionTestUtils.setField(semanticService, "userGroupService", userGroupService);
ReflectionTestUtils.setField(semanticService, "expandableMeasureUtil",
new ExpandableMeasureUtil((model, ccDesc) -> {
- String ccExpression = KapQueryUtil.massageComputedColumn(model, model.getProject(), ccDesc,
+ String ccExpression = QueryUtil.massageComputedColumn(model, model.getProject(), ccDesc,
AclPermissionUtil.prepareQueryContextACLInfo(model.getProject(),
semanticService.getCurrentUserGroups()));
ccDesc.setInnerExpression(ccExpression);
diff --git a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/builder/SegmentFlatTable.scala b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/builder/SegmentFlatTable.scala
index d93cb71de3..bd590e6104 100644
--- a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/builder/SegmentFlatTable.scala
+++ b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/builder/SegmentFlatTable.scala
@@ -18,9 +18,10 @@
package org.apache.kylin.engine.spark.builder
-import java.util.concurrent.{CountDownLatch, TimeUnit}
-import java.util.{Locale, Objects, Timer, TimerTask}
import com.google.common.collect.Sets
+import org.apache.commons.lang3.StringUtils
+import org.apache.kylin.common.util.HadoopUtil
+import org.apache.kylin.common.{KapConfig, KylinConfig}
import org.apache.kylin.engine.spark.builder.DFBuilderHelper._
import org.apache.kylin.engine.spark.job.NSparkCubingUtil._
import org.apache.kylin.engine.spark.job.{FiltersUtil, TableMetaManager}
@@ -28,18 +29,16 @@ import org.apache.kylin.engine.spark.model.SegmentFlatTableDesc
import org.apache.kylin.engine.spark.utils.LogEx
import org.apache.kylin.engine.spark.utils.SparkDataSource._
import org.apache.kylin.metadata.cube.model.NDataSegment
-import org.apache.kylin.metadata.model.{NDataModel, NTableMetadataManager}
-import org.apache.commons.lang3.StringUtils
-import org.apache.kylin.common.{KapConfig, KylinConfig}
-import org.apache.kylin.common.util.HadoopUtil
import org.apache.kylin.metadata.model._
-import org.apache.kylin.query.util.KapQueryUtil
+import org.apache.kylin.query.util.QueryUtil
import org.apache.spark.sql._
import org.apache.spark.sql.functions.{col, expr}
import org.apache.spark.sql.types.StructField
import org.apache.spark.sql.util.SparderTypeUtil
import org.apache.spark.utils.ProxyThreadUtils
+import java.util.concurrent.{CountDownLatch, TimeUnit}
+import java.util.{Locale, Objects, Timer, TimerTask}
import scala.collection.JavaConverters._
import scala.collection.mutable
import scala.collection.parallel.ForkJoinTaskSupport
@@ -138,13 +137,13 @@ class SegmentFlatTable(private val sparkSession: SparkSession, //
}
/**
- * If need to build and encode dict columns, then
- * 1. try best to build in fact-table.
- * 2. try best to build in lookup-tables (without cc dict).
- * 3. try to build in fact-table.
- *
- * CC in lookup-tables MUST be built in flat-table.
- */
+ * If need to build and encode dict columns, then
+ * 1. try best to build in fact-table.
+ * 2. try best to build in lookup-tables (without cc dict).
+ * 3. try to build in fact-table.
+ *
+ * CC in lookup-tables MUST be built in flat-table.
+ */
val (dictCols, encodeCols, dictColsWithoutCc, encodeColsWithoutCc) = prepareForDict()
val factTable = buildDictIfNeed(factTableDS, dictCols, encodeCols)
@@ -257,7 +256,7 @@ class SegmentFlatTable(private val sparkSession: SparkSession, //
logInfo(s"No available FILTER-CONDITION segment $segmentId")
return originDS
}
- val expression = KapQueryUtil.massageExpression(dataModel, project, //
+ val expression = QueryUtil.massageExpression(dataModel, project, //
dataModel.getFilterCondition, null)
val converted = replaceDot(expression, dataModel)
val condition = s" (1=1) AND ($converted)"
@@ -368,13 +367,13 @@ class SegmentFlatTable(private val sparkSession: SparkSession, //
// If fact table is a view and its snapshot exists, that will benefit.
logInfo(s"Load source table ${tableRef.getTableIdentity}")
val tableDescCopy = tableRef.getTableDesc
- if(tableDescCopy.isTransactional || tableDescCopy.isRangePartition) {
+ if (tableDescCopy.isTransactional || tableDescCopy.isRangePartition) {
val model = tableRef.getModel
- if(Objects.nonNull(model)) {
+ if (Objects.nonNull(model)) {
tableDescCopy.setPartitionDesc(model.getPartitionDesc)
}
- if(Objects.nonNull(segmentRange) && Objects.nonNull(segmentRange.getStart) && Objects.nonNull(segmentRange.getEnd)) {
+ if (Objects.nonNull(segmentRange) && Objects.nonNull(segmentRange.getStart) && Objects.nonNull(segmentRange.getEnd)) {
sparkSession.table(tableDescCopy, segmentRange.getStart.toString, segmentRange.getEnd.toString).alias(tableRef.getAlias)
} else {
sparkSession.table(tableDescCopy).alias(tableRef.getAlias)
diff --git a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/FlatTableHelper.scala b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/FlatTableHelper.scala
index 7213f53e71..d9af449653 100644
--- a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/FlatTableHelper.scala
+++ b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/FlatTableHelper.scala
@@ -18,19 +18,19 @@
package org.apache.kylin.engine.spark.job
-import org.apache.kylin.engine.spark.builder.CreateFlatTable.replaceDot
import org.apache.commons.lang3.StringUtils
+import org.apache.kylin.engine.spark.builder.CreateFlatTable.replaceDot
import org.apache.kylin.metadata.model.IJoinedFlatTableDesc
-import org.apache.kylin.query.util.KapQueryUtil
+import org.apache.kylin.query.util.QueryUtil
import org.apache.spark.internal.Logging
import org.apache.spark.sql.{Dataset, Row}
object FlatTableHelper extends Logging {
def applyPartitionDesc(
- flatTable: IJoinedFlatTableDesc,
- ds: Dataset[Row],
- needReplaceDot: Boolean): Dataset[Row] = {
+ flatTable: IJoinedFlatTableDesc,
+ ds: Dataset[Row],
+ needReplaceDot: Boolean): Dataset[Row] = {
var afterFilter = ds
val model = flatTable.getDataModel
@@ -50,15 +50,15 @@ object FlatTableHelper extends Logging {
}
def applyFilterCondition(
- flatTable: IJoinedFlatTableDesc,
- ds: Dataset[Row],
- needReplaceDot: Boolean): Dataset[Row] = {
+ flatTable: IJoinedFlatTableDesc,
+ ds: Dataset[Row],
+ needReplaceDot: Boolean): Dataset[Row] = {
var afterFilter = ds
val model = flatTable.getDataModel
if (StringUtils.isNotBlank(model.getFilterCondition)) {
var filterCond = model.getFilterCondition
- filterCond = KapQueryUtil.massageExpression(model, model.getProject, filterCond, null);
+ filterCond = QueryUtil.massageExpression(model, model.getProject, filterCond, null);
if (needReplaceDot) filterCond = replaceDot(filterCond, model)
filterCond = s" (1=1) AND (" + filterCond + s")"
logInfo(s"Filter condition is $filterCond")
diff --git a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/stage/build/FlatTableAndDictBase.scala b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/stage/build/FlatTableAndDictBase.scala
index d80e1796a8..f06bee49c2 100644
--- a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/stage/build/FlatTableAndDictBase.scala
+++ b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/stage/build/FlatTableAndDictBase.scala
@@ -35,7 +35,7 @@ import org.apache.kylin.engine.spark.utils.LogEx
import org.apache.kylin.engine.spark.utils.SparkDataSource._
import org.apache.kylin.metadata.cube.model.NDataSegment
import org.apache.kylin.metadata.model._
-import org.apache.kylin.query.util.KapQueryUtil
+import org.apache.kylin.query.util.QueryUtil
import org.apache.spark.sql.KapFunctions.dict_encode_v3
import org.apache.spark.sql._
import org.apache.spark.sql.functions.{col, expr}
@@ -273,7 +273,7 @@ abstract class FlatTableAndDictBase(private val jobContext: SegmentJob,
logInfo(s"No available FILTER-CONDITION segment $segmentId")
return originDS
}
- val expression = KapQueryUtil.massageExpression(dataModel, project, //
+ val expression = QueryUtil.massageExpression(dataModel, project, //
dataModel.getFilterCondition, null)
val converted = replaceDot(expression, dataModel)
val condition = s" (1=1) AND ($converted)"
@@ -503,7 +503,7 @@ abstract class FlatTableAndDictBase(private val jobContext: SegmentJob,
buildDict(table, dictCols)
}
- if(config.isV3DictEnable) {
+ if (config.isV3DictEnable) {
buildV3DictIfNeeded(table, encodeCols)
} else {
encodeColumn(table, encodeCols)
@@ -513,14 +513,14 @@ abstract class FlatTableAndDictBase(private val jobContext: SegmentJob,
protected def buildV3DictIfNeeded(table: Dataset[Row], dictCols: Set[TblColRef]): Dataset[Row] = {
logInfo("Build v3 dict if needed.")
val matchedCols = selectColumnsInTable(table, dictCols)
- val cols = matchedCols.map{ dictColumn =>
+ val cols = matchedCols.map { dictColumn =>
val wrapDictCol = DictionaryBuilder.wrapCol(dictColumn)
dict_encode_v3(col(wrapDictCol)).alias(wrapDictCol + "_KE_ENCODE")
}.toSeq
val dictPlan = table
.select(table.schema.map(ty => col(ty.name)) ++ cols: _*)
- .queryExecution
- .analyzed
+ .queryExecution
+ .analyzed
val encodePlan = DictionaryBuilder.buildGlobalDict(project, sparkSession, dictPlan)
SparkInternalAgent.getDataFrame(sparkSession, encodePlan)
}
diff --git a/src/spark-project/sparder/src/main/scala/org/apache/kylin/query/pushdown/SparkSqlClient.scala b/src/spark-project/sparder/src/main/scala/org/apache/kylin/query/pushdown/SparkSqlClient.scala
index 82cd8e34fb..5b6d20c715 100644
--- a/src/spark-project/sparder/src/main/scala/org/apache/kylin/query/pushdown/SparkSqlClient.scala
+++ b/src/spark-project/sparder/src/main/scala/org/apache/kylin/query/pushdown/SparkSqlClient.scala
@@ -18,10 +18,17 @@
package org.apache.kylin.query.pushdown
+import com.google.common.collect.ImmutableList
import io.kyligence.kap.guava20.shaded.common.collect.Lists
+import org.apache.commons.lang3.StringUtils
+import org.apache.kylin.common.util.{DateFormat, HadoopUtil, Pair}
+import org.apache.kylin.common.{KapConfig, KylinConfig, QueryContext}
import org.apache.kylin.metadata.project.NProjectManager
import org.apache.kylin.metadata.query.StructField
-import org.apache.commons.lang3.StringUtils
+import org.apache.kylin.query.mask.QueryResultMasks
+import org.apache.kylin.query.runtime.plan.QueryToExecutionIDCache
+import org.apache.kylin.query.runtime.plan.ResultPlan.saveAsyncQueryResult
+import org.apache.kylin.query.util.{QueryUtil, SparkJobTrace}
import org.apache.spark.network.util.JavaUtils
import org.apache.spark.sql.hive.QueryMetricUtils
import org.apache.spark.sql.hive.utils.ResourceDetectUtils
@@ -31,17 +38,6 @@ import org.slf4j.{Logger, LoggerFactory}
import java.sql.Timestamp
import java.util.{UUID, List => JList}
-import com.google.common.collect.ImmutableList
-import org.apache.kylin.common.exception.KylinTimeoutException
-import org.apache.kylin.common.{KapConfig, KylinConfig, QueryContext}
-import org.apache.kylin.common.util.{DateFormat, HadoopUtil, Pair}
-import org.apache.kylin.query.SlowQueryDetector
-import org.apache.kylin.query.exception.UserStopQueryException
-import org.apache.kylin.query.mask.QueryResultMasks
-import org.apache.kylin.query.runtime.plan.QueryToExecutionIDCache
-import org.apache.kylin.query.runtime.plan.ResultPlan.saveAsyncQueryResult
-import org.apache.kylin.query.util.SparkJobTrace
-
import scala.collection.JavaConverters._
import scala.collection.{immutable, mutable}
@@ -91,8 +87,8 @@ object SparkSqlClient {
try {
val basePartitionSize = config.getBaseShufflePartitionSize
val paths = ResourceDetectUtils.getPaths(df.queryExecution.sparkPlan)
- val sourceTableSize = ResourceDetectUtils.getResourceSize(SparderEnv.getHadoopConfiguration(), config.isConcurrencyFetchDataSourceSize,
- paths: _*) + "b"
+ val sourceTableSize = ResourceDetectUtils.getResourceSize(SparderEnv.getHadoopConfiguration(),
+ config.isConcurrencyFetchDataSourceSize, paths: _*) + "b"
val partitions = Math.max(1, JavaUtils.byteStringAsMb(sourceTableSize) / basePartitionSize).toString
df.sparkSession.sessionState.conf.setLocalProperty("spark.sql.shuffle.partitions", partitions)
QueryContext.current().setShufflePartitions(partitions.toInt)
@@ -125,7 +121,6 @@ object SparkSqlClient {
val results = df.toIterator()
val resultRows = results._1
val resultSize = results._2
-
if (config.isQuerySparkJobTraceEnabled) jobTrace.jobFinished()
val fieldList = df.schema.map(field => SparderTypeUtil.convertSparkFieldToJavaField(field)).asJava
val (scanRows, scanBytes) = QueryMetricUtils.collectScanMetrics(df.queryExecution.executedPlan)
@@ -137,10 +132,22 @@ object SparkSqlClient {
QueryContext.current().getMetrics.setQueryTaskCount(taskCount)
(
() => new java.util.Iterator[JList[String]] {
+ /*
+ * After fetching a batch of 1000, checks whether the query thread is interrupted.
+ */
+ val checkInterruptSize = 1000;
+ var readRowSize = 0;
+
override def hasNext: Boolean = resultRows.hasNext
override def next(): JList[String] = {
- resultRows.next().toSeq.map(rawValueToString(_)).asJava
+ val row = resultRows.next()
+ readRowSize += 1;
+ if (readRowSize % checkInterruptSize == 0) {
+ QueryUtil.checkThreadInterrupted("Interrupted at the stage of collecting result in SparkSqlClient.",
+ "Current step: Collecting dataset of push-down.")
+ }
+ row.toSeq.map(rawValueToString(_)).asJava
}
},
resultSize,
@@ -151,15 +158,10 @@ object SparkSqlClient {
if (e.isInstanceOf[InterruptedException]) {
Thread.currentThread.interrupt()
ss.sparkContext.cancelJobGroup(jobGroup)
- if (SlowQueryDetector.getRunningQueries.get(Thread.currentThread()).isStopByUser) {
- throw new UserStopQueryException("")
- }
- QueryContext.current.getQueryTagInfo.setTimeout(true)
- logger.info("Query timeout ", e)
- throw new KylinTimeoutException("The query exceeds the set time limit of "
- + KylinConfig.getInstanceFromEnv.getQueryTimeoutSeconds + "s. Current step: Collecting dataset for push-down. ")
+ QueryUtil.checkThreadInterrupted("Interrupted at the stage of collecting result in SparkSqlClient.",
+ "Current step: Collecting dataset of push-down.")
}
- else throw e
+ throw e
} finally {
QueryContext.current().setExecutionID(QueryToExecutionIDCache.getQueryExecutionID(QueryContext.current().getQueryId))
df.sparkSession.sessionState.conf.setLocalProperty("spark.sql.shuffle.partitions", null)
@@ -171,7 +173,8 @@ object SparkSqlClient {
case null => null
case value: Timestamp => DateFormat.castTimestampToString(value.getTime)
case value: String => if (wrapped) "\"" + value + "\"" else value
- case value: mutable.WrappedArray.ofRef[Any] => value.array.map(v => rawValueToString(v, true)).mkString("[", ",", "]")
+ case value: mutable.WrappedArray[AnyVal] => value.array.map(v => rawValueToString(v, true)).mkString("[", ",", "]")
+ case value: mutable.WrappedArray.ofRef[AnyRef] => value.array.map(v => rawValueToString(v, true)).mkString("[", ",", "]")
case value: immutable.Map[Any, Any] =>
value.map(p => rawValueToString(p._1, true) + ":" + rawValueToString(p._2, true)).mkString("{", ",", "}")
case value: Any => value.toString
diff --git a/src/spark-project/sparder/src/main/scala/org/apache/kylin/query/runtime/plan/ResultPlan.scala b/src/spark-project/sparder/src/main/scala/org/apache/kylin/query/runtime/plan/ResultPlan.scala
index b227e9162b..7ecc408674 100644
--- a/src/spark-project/sparder/src/main/scala/org/apache/kylin/query/runtime/plan/ResultPlan.scala
+++ b/src/spark-project/sparder/src/main/scala/org/apache/kylin/query/runtime/plan/ResultPlan.scala
@@ -20,19 +20,20 @@ package org.apache.kylin.query.runtime.plan
import com.google.common.cache.{Cache, CacheBuilder}
import io.kyligence.kap.secondstorage.SecondStorageUtil
-
import org.apache.calcite.rel.`type`.RelDataType
-import org.apache.kylin.common.exception.{KylinTimeoutException, NewQueryRefuseException}
+import org.apache.hadoop.fs.Path
+import org.apache.kylin.common.exception.NewQueryRefuseException
import org.apache.kylin.common.util.{HadoopUtil, RandomUtil}
import org.apache.kylin.common.{KapConfig, KylinConfig, QueryContext}
import org.apache.kylin.engine.spark.utils.LogEx
import org.apache.kylin.metadata.query.{BigQueryThresholdUpdater, StructField}
import org.apache.kylin.metadata.state.QueryShareStateManager
-import org.apache.kylin.query.SlowQueryDetector
import org.apache.kylin.query.engine.RelColumnMetaDataExtractor
import org.apache.kylin.query.engine.exec.ExecuteResult
-import org.apache.kylin.query.exception.UserStopQueryException
-import org.apache.kylin.query.util.{AsyncQueryUtil, SparkJobTrace, SparkQueryJobManager}
+import org.apache.kylin.query.relnode.OLAPContext
+import org.apache.kylin.query.util.{AsyncQueryUtil, QueryUtil, SparkJobTrace, SparkQueryJobManager}
+import org.apache.poi.xssf.usermodel.XSSFWorkbook
+import org.apache.spark.SparkConf
import org.apache.spark.sql.execution._
import org.apache.spark.sql.hive.QueryMetricUtils
import org.apache.spark.sql.util.SparderTypeUtil
@@ -40,11 +41,6 @@ import org.apache.spark.sql.{DataFrame, SaveMode, SparderEnv}
import java.io.{File, FileOutputStream}
import java.util
-import org.apache.hadoop.fs.Path
-import org.apache.kylin.query.relnode.OLAPContext
-import org.apache.poi.xssf.usermodel.XSSFWorkbook
-import org.apache.spark.SparkConf
-
import java.util.concurrent.atomic.AtomicLong
import scala.collection.JavaConverters._
import scala.collection.mutable
@@ -104,7 +100,7 @@ object ResultPlan extends LogEx {
// judge whether to refuse the new big query
logDebug(s"Total source scan rows: $sumOfSourceScanRows")
- if(QueryShareStateManager.isShareStateSwitchEnabled
+ if (QueryShareStateManager.isShareStateSwitchEnabled
&& sumOfSourceScanRows >= bigQueryThreshold
&& SparkQueryJobManager.isNewBigQueryRefuse) {
QueryContext.current().getQueryTagInfo.setRefused(true)
@@ -158,17 +154,14 @@ object ResultPlan extends LogEx {
}
}, resultSize)
} catch {
- case e: InterruptedException =>
- Thread.currentThread.interrupt()
- sparkContext.cancelJobGroup(jobGroup)
- if (SlowQueryDetector.getRunningQueries.get(Thread.currentThread()).isStopByUser) {
- throw new UserStopQueryException("")
+ case e: Throwable =>
+ if (e.isInstanceOf[InterruptedException]) {
+ Thread.currentThread.interrupt()
+ sparkContext.cancelJobGroup(jobGroup)
+ QueryUtil.checkThreadInterrupted("Interrupted at the stage of collecting result in ResultPlan.",
+ "Current step: Collecting dataset for sparder.")
}
- QueryContext.current().getQueryTagInfo.setTimeout(true)
- logWarning(s"Query timeouts after: ${KylinConfig.getInstanceFromEnv.getQueryTimeoutSeconds}s")
- throw new KylinTimeoutException("The query exceeds the set time limit of "
- + KylinConfig.getInstanceFromEnv.getQueryTimeoutSeconds + "s. Current step: Collecting dataset for sparder. ")
- case e: Throwable => throw e
+ throw e
} finally {
QueryContext.current().setExecutionID(QueryToExecutionIDCache.getQueryExecutionID(queryId))
}
@@ -281,7 +274,7 @@ object ResultPlan extends LogEx {
sparkContext.setJobGroup(jobGroup,
QueryContext.current().getMetrics.getCorrectedSql,
interruptOnCancel = true)
- if(kapConfig.isQueryLimitEnabled && SparderEnv.isSparkExecutorResourceLimited(sparkContext.getConf)) {
+ if (kapConfig.isQueryLimitEnabled && SparderEnv.isSparkExecutorResourceLimited(sparkContext.getConf)) {
sparkContext.setLocalProperty(SPARK_SCHEDULER_POOL, "async_query_tasks")
}
df.sparkSession.sparkContext.setLocalProperty(QueryToExecutionIDCache.KYLIN_QUERY_EXECUTION_ID, queryExecutionId)
@@ -298,7 +291,7 @@ object ResultPlan extends LogEx {
newDf = newDf.withColumnRenamed(oldColumnNames.apply(i), columnNames.get(i))
}
newDf.write.option("timestampFormat", dateTimeFormat).option("encoding", encode)
- .option("charset", "utf-8").mode(SaveMode.Append).json(path)
+ .option("charset", "utf-8").mode(SaveMode.Append).json(path)
case "parquet" =>
val sqlContext = SparderEnv.getSparkSession.sqlContext
sqlContext.setConf("spark.sql.parquet.writeLegacyFormat", "true")
@@ -311,10 +304,10 @@ object ResultPlan extends LogEx {
sqlContext.setConf("spark.sql.parquet.writeLegacyFormat", "false")
case "csv" =>
df.write
- .option("timestampFormat", dateTimeFormat)
- .option("encoding", encode)
- .option("dateFormat", "yyyy-MM-dd")
- .option("charset", "utf-8").mode(SaveMode.Append).csv(path)
+ .option("timestampFormat", dateTimeFormat)
+ .option("encoding", encode)
+ .option("dateFormat", "yyyy-MM-dd")
+ .option("charset", "utf-8").mode(SaveMode.Append).csv(path)
case "xlsx" => {
val queryId = QueryContext.current().getQueryId
val file = new File(queryId + ".xlsx")
diff --git a/src/spark-project/spark-it/src/test/scala/org/apache/kylin/common/CompareSupport.scala b/src/spark-project/spark-it/src/test/scala/org/apache/kylin/common/CompareSupport.scala
index 9206ef1bef..29a36ab8ba 100644
--- a/src/spark-project/spark-it/src/test/scala/org/apache/kylin/common/CompareSupport.scala
+++ b/src/spark-project/spark-it/src/test/scala/org/apache/kylin/common/CompareSupport.scala
@@ -17,10 +17,10 @@
package org.apache.kylin.common
-import org.apache.kylin.util.ExecAndComp.EnhancedQueryResult
-import org.apache.kylin.util.{ExecAndComp, QueryResultComparator}
import io.netty.util.internal.ThrowableUtil
import org.apache.kylin.query.engine.data.QueryResult
+import org.apache.kylin.util.ExecAndComp.EnhancedQueryResult
+import org.apache.kylin.util.{ExecAndComp, QueryResultComparator}
import org.scalatest.Suite
trait CompareSupport extends QuerySupport {
diff --git a/src/spark-project/spark-it/src/test/scala/org/apache/kylin/common/QuerySupport.scala b/src/spark-project/spark-it/src/test/scala/org/apache/kylin/common/QuerySupport.scala
index 2f26e347f7..358f3e7ad4 100644
--- a/src/spark-project/spark-it/src/test/scala/org/apache/kylin/common/QuerySupport.scala
+++ b/src/spark-project/spark-it/src/test/scala/org/apache/kylin/common/QuerySupport.scala
@@ -17,18 +17,19 @@
package org.apache.kylin.common
-import java.util.Locale
import org.apache.commons.lang3.StringUtils
-import org.apache.kylin.common.KylinConfig
import org.apache.kylin.common.util.Unsafe
+import org.apache.kylin.metadata.project.NProjectManager
import org.apache.kylin.query.engine.QueryExec
-import org.apache.kylin.query.util.{KapQueryUtil, QueryParams}
+import org.apache.kylin.query.util.{QueryParams, QueryUtil}
import org.apache.spark.internal.Logging
import org.apache.spark.sql.common.{SharedSparkSession, SparderQueryTest}
import org.apache.spark.sql.udf.UdfManager
import org.apache.spark.sql.{DataFrame, SparderEnv}
import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach, Suite}
+import java.util.Locale
+
trait QuerySupport
extends BeforeAndAfterAll
with BeforeAndAfterEach
@@ -58,9 +59,9 @@ trait QuerySupport
val prevRunLocalConf = Unsafe.setProperty("kylin.query.engine.run-constant-query-locally", "FALSE")
try {
val queryExec = new QueryExec(project, KylinConfig.getInstanceFromEnv)
- val queryParams = new QueryParams(KapQueryUtil.getKylinConfig(project), sql, project,
+ val queryParams = new QueryParams(NProjectManager.getProjectConfig(project), sql, project,
0, 0, queryExec.getDefaultSchemaName, true)
- val convertedSql = KapQueryUtil.massageSql(queryParams)
+ val convertedSql = QueryUtil.massageSql(queryParams)
queryExec.executeQuery(convertedSql)
} finally {
if (prevRunLocalConf == null) {
diff --git a/src/spark-project/spark-it/src/test/scala/org/apache/kylin/common/SSSource.scala b/src/spark-project/spark-it/src/test/scala/org/apache/kylin/common/SSSource.scala
index 2b7d5d220f..b23641993c 100644
--- a/src/spark-project/spark-it/src/test/scala/org/apache/kylin/common/SSSource.scala
+++ b/src/spark-project/spark-it/src/test/scala/org/apache/kylin/common/SSSource.scala
@@ -18,10 +18,10 @@
package org.apache.kylin.common
import com.google.common.base.Preconditions
-import org.apache.kylin.common.KylinConfig
import org.apache.kylin.common.util.TempMetadataBuilder
import org.apache.kylin.metadata.model.NTableMetadataManager
-import org.apache.kylin.query.util.{KapQueryUtil, QueryParams}
+import org.apache.kylin.metadata.project.NProjectManager
+import org.apache.kylin.query.util.{QueryParams, QueryUtil}
import org.apache.spark.sql.common.{LocalMetadata, SharedSparkSession}
import org.apache.spark.sql.execution.utils.SchemaProcessor
import org.scalatest.Suite
@@ -76,7 +76,7 @@ trait SSSource extends SharedSparkSession with LocalMetadata {
.replaceAll("DEFAULT\\.", "")
.replaceAll("\"DEFAULT\"\\.", "")
val queryParams = new QueryParams("default", sqlForSpark, "DEFAULT", false)
- queryParams.setKylinConfig(KapQueryUtil.getKylinConfig("default"))
- KapQueryUtil.massagePushDownSql(queryParams)
+ queryParams.setKylinConfig(NProjectManager.getProjectConfig("default"))
+ QueryUtil.massagePushDownSql(queryParams)
}
}
diff --git a/src/spark-project/spark-it/src/test/scala/org/apache/kylin/common/YarnSupport.scala b/src/spark-project/spark-it/src/test/scala/org/apache/kylin/common/YarnSupport.scala
index 66111cd370..477fc45def 100644
--- a/src/spark-project/spark-it/src/test/scala/org/apache/kylin/common/YarnSupport.scala
+++ b/src/spark-project/spark-it/src/test/scala/org/apache/kylin/common/YarnSupport.scala
@@ -17,7 +17,6 @@
package org.apache.kylin.common
-import org.apache.kylin.common.KylinConfig
import org.apache.kylin.common.util.ClassUtil
import org.apache.spark.internal.Logging
import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach, Suite}
[kylin] 14/17: KYLIN-5397 Support sum_lc function
Posted by xx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 60ad2aa86de528209b898c2d2bb343bcec1c3d7e
Author: Bowen Song <bo...@kyligence.io>
AuthorDate: Thu Nov 10 11:23:54 2022 +0800
KYLIN-5397 Support sum_lc function
* Support sum_lc function
* Remove wrap sum_lc type
* Remove rewrite logic, and unnecessary check
* fix part ut
---
.../kylin/rest/service/AccessServiceTest.java | 9 +-
.../kylin/rest/service/AclTCRServiceTest.java | 4 +-
.../java/org/apache/kylin/common/msg/Message.java | 2 +-
.../org/apache/kylin/common/util/DateFormat.java | 29 ++-
.../apache/kylin/common/util/DateFormatTest.java | 51 ++++
.../apache/kylin/measure/MeasureTypeFactory.java | 2 +
.../apache/kylin/measure/sumlc/SumLCCounter.java | 96 ++++++++
.../kylin/measure/sumlc/SumLCMeasureType.java | 172 +++++++++++++
.../apache/kylin/metadata/datatype/DataType.java | 10 +-
.../apache/kylin/metadata/model/FunctionDesc.java | 68 ++++--
.../metadata/model/util/FunctionDescTest.java | 12 +-
.../metadata/project/NProjectManagerTest.java | 5 +-
.../localmeta/metadata/_global/project/sum_lc.json | 35 +++
.../f35f2937-9e4d-347a-7465-d64df939e7d6.json | 13 +
.../f35f2937-9e4d-347a-7465-d64df939e7d6.json | 43 ++++
.../f35f2937-9e4d-347a-7465-d64df939e7d6.json | 267 +++++++++++++++++++++
.../metadata/sum_lc/table/SSB.SUMLC_EXTEND_4X.json | 78 ++++++
.../kylin/rest/service/ProjectServiceTest.java | 8 +-
.../kylin/query/relnode/KapAggregateRel.java | 21 +-
.../kylin/query/relnode/OLAPAggregateRel.java | 2 +-
.../rest/service/QueryHistoryServiceTest.java | 2 +-
.../kylin/query/engine/QueryRoutingEngine.java | 15 ++
.../kylin/query/engine/AsyncQueryJobTest.java | 6 +-
.../kylin/query/engine/QueryRoutingEngineTest.java | 16 ++
.../kylin/engine/spark/job/CuboidAggregator.scala | 41 ++--
.../kylin/query/runtime/plan/AggregatePlan.scala | 11 +-
.../scala/org/apache/spark/sql/KapFunctions.scala | 14 +-
.../sql/catalyst/expressions/ExpressionUtils.scala | 31 ++-
.../sql/catalyst/expressions/KapExpresssions.scala | 64 ++++-
.../org/apache/spark/sql/udf/SparderAggFun.scala | 11 +-
.../apache/spark/sql/LayoutEntityConverter.scala | 6 +-
.../spark/sql/udaf/NullSafeValueSerializer.scala | 30 ++-
.../scala/org/apache/spark/sql/udaf/SumLC.scala | 191 +++++++++++++++
33 files changed, 1256 insertions(+), 109 deletions(-)
diff --git a/src/common-service/src/test/java/org/apache/kylin/rest/service/AccessServiceTest.java b/src/common-service/src/test/java/org/apache/kylin/rest/service/AccessServiceTest.java
index fb102f63de..b0f9142756 100644
--- a/src/common-service/src/test/java/org/apache/kylin/rest/service/AccessServiceTest.java
+++ b/src/common-service/src/test/java/org/apache/kylin/rest/service/AccessServiceTest.java
@@ -48,6 +48,7 @@ import org.apache.kylin.common.persistence.AclEntity;
import org.apache.kylin.common.util.NLocalFileMetadataTestCase;
import org.apache.kylin.metadata.project.NProjectManager;
import org.apache.kylin.metadata.project.ProjectInstance;
+import org.apache.kylin.metadata.user.ManagedUser;
import org.apache.kylin.rest.constant.Constant;
import org.apache.kylin.rest.request.AccessRequest;
import org.apache.kylin.rest.request.GlobalAccessRequest;
@@ -98,8 +99,6 @@ import com.fasterxml.jackson.core.JsonProcessingException;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
-import org.apache.kylin.metadata.user.ManagedUser;
-
@RunWith(PowerMockRunner.class)
@PrepareForTest({ SpringContext.class, UserGroupInformation.class, KylinConfig.class, NProjectManager.class })
public class AccessServiceTest extends NLocalFileMetadataTestCase {
@@ -597,14 +596,14 @@ public class AccessServiceTest extends NLocalFileMetadataTestCase {
@Test
public void testGetGrantedProjectsOfUser() throws IOException {
List<String> result = accessService.getGrantedProjectsOfUser("ADMIN");
- assertEquals(27, result.size());
+ assertEquals(28, result.size());
}
@Test
public void testGetGrantedProjectsOfUserOrGroup() throws IOException {
// admin user
List<String> result = accessService.getGrantedProjectsOfUserOrGroup("ADMIN", true);
- assertEquals(27, result.size());
+ assertEquals(28, result.size());
// normal user
result = accessService.getGrantedProjectsOfUserOrGroup("ANALYST", true);
@@ -786,7 +785,7 @@ public class AccessServiceTest extends NLocalFileMetadataTestCase {
}
@Test
- public void testAclWithUnNaturalOrderUpdate() throws IOException{
+ public void testAclWithUnNaturalOrderUpdate() throws IOException {
AclEntity ae = accessService.getAclEntity(AclEntityType.PROJECT_INSTANCE,
"1eaca32a-a33e-4b69-83dd-0bb8b1f8c91b");
diff --git a/src/common-service/src/test/java/org/apache/kylin/rest/service/AclTCRServiceTest.java b/src/common-service/src/test/java/org/apache/kylin/rest/service/AclTCRServiceTest.java
index bb994174a9..9a0ee62d7d 100644
--- a/src/common-service/src/test/java/org/apache/kylin/rest/service/AclTCRServiceTest.java
+++ b/src/common-service/src/test/java/org/apache/kylin/rest/service/AclTCRServiceTest.java
@@ -311,7 +311,7 @@ public class AclTCRServiceTest extends NLocalFileMetadataTestCase {
}
private SensitiveDataMask.MaskType getColumnDataMask(AclTCRRequest acl, String database, String table,
- String column) {
+ String column) {
if (acl.getDatabaseName().equals(database)) {
for (val tb : acl.getTables()) {
if (tb.getTableName().equals(table)) {
@@ -1316,7 +1316,7 @@ public class AclTCRServiceTest extends NLocalFileMetadataTestCase {
Mockito.when(userService.isGlobalAdmin("ADMIN")).thenReturn(true);
List<SidPermissionWithAclResponse> responses = accessService.getUserOrGroupAclPermissions(projects, "ADMIN",
true);
- Assert.assertEquals(27, responses.size());
+ Assert.assertEquals(28, responses.size());
Assert.assertTrue(responses.stream().allMatch(response -> "ADMIN".equals(response.getProjectPermission())));
// test normal group
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/msg/Message.java b/src/core-common/src/main/java/org/apache/kylin/common/msg/Message.java
index 365474db28..2048ce9bc1 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/msg/Message.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/msg/Message.java
@@ -1015,7 +1015,7 @@ public class Message {
}
public String getInvalidTimeFormat() {
- return "Can’t set the time partition column. The values of the selected column is not time formatted. Please select again.";
+ return "Can’t set the time partition column. The values of the selected column is not time formatted: {%s}. Please select again.";
}
public String getSegmentMergeStorageCheckError() {
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/util/DateFormat.java b/src/core-common/src/main/java/org/apache/kylin/common/util/DateFormat.java
index e5391e3b16..fd8dd243fa 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/util/DateFormat.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/util/DateFormat.java
@@ -64,6 +64,10 @@ public class DateFormat {
public static final String DEFAULT_DATETIME_PATTERN_WITH_MILLISECONDS = "yyyy-MM-dd HH:mm:ss.SSS";
public static final String DEFAULT_DATETIME_PATTERN_WITH_TIMEZONE = "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'";
+ private static final int NANOS_TIMESTAMP_LENGTH = 16;
+ private static final int MILLIS_TIMESTAMP_LENGTH = 13;
+ private static final int SECONDS_TIMESTAMP_LENGTH = 10;
+
static final private Map<String, FastDateFormat> formatMap = new ConcurrentHashMap<String, FastDateFormat>();
private static final Map<String, String> dateFormatRegex = Maps.newHashMap();
@@ -210,13 +214,26 @@ public class DateFormat {
return stringToDate(str, regexToPattern.getValue()).getTime();
}
- // try parse it as days to epoch
try {
- long daysToEpoch = Long.parseLong(str);
- return daysToEpoch * 24 * 60 * 60 * 1000;
+ long strToDigit = Long.parseLong(str);
+ if (strToDigit > 0) {
+ if (str.length() == NANOS_TIMESTAMP_LENGTH) {
+ return strToDigit / 1000;
+ } else if (str.length() == MILLIS_TIMESTAMP_LENGTH) {
+ return strToDigit;
+ } else if (str.length() == SECONDS_TIMESTAMP_LENGTH) {
+ return strToDigit * 1000;
+ } else {
+ // try parse it as days to epoch
+ return strToDigit * 24 * 60 * 60 * 1000;
+ }
+ }
} catch (NumberFormatException e) {
+ throw new KylinException(INVALID_TIME_PARTITION_COLUMN,
+ String.format(Locale.ROOT, MsgPicker.getMsg().getInvalidTimeFormat(), str), e);
}
- throw new KylinException(INVALID_TIME_PARTITION_COLUMN, MsgPicker.getMsg().getInvalidTimeFormat());
+ throw new KylinException(INVALID_TIME_PARTITION_COLUMN,
+ String.format(Locale.ROOT, MsgPicker.getMsg().getInvalidTimeFormat(), str));
}
public static boolean isSupportedDateFormat(String dateStr) {
@@ -244,7 +261,8 @@ public class DateFormat {
if (sampleData.matches(patternMap.getKey()))
return patternMap.getValue();
}
- throw new KylinException(INVALID_TIME_PARTITION_COLUMN, MsgPicker.getMsg().getInvalidTimeFormat());
+ throw new KylinException(INVALID_TIME_PARTITION_COLUMN,
+ String.format(Locale.ROOT, MsgPicker.getMsg().getInvalidTimeFormat(), sampleData));
}
/**
@@ -291,4 +309,5 @@ public class DateFormat {
|| DEFAULT_DATETIME_PATTERN_WITH_MILLISECONDS.equals(format)
|| DEFAULT_DATETIME_PATTERN_WITH_TIMEZONE.equals(format));
}
+
}
diff --git a/src/core-common/src/test/java/org/apache/kylin/common/util/DateFormatTest.java b/src/core-common/src/test/java/org/apache/kylin/common/util/DateFormatTest.java
index 9f5364e8cc..0387906235 100644
--- a/src/core-common/src/test/java/org/apache/kylin/common/util/DateFormatTest.java
+++ b/src/core-common/src/test/java/org/apache/kylin/common/util/DateFormatTest.java
@@ -24,12 +24,14 @@ import java.time.format.DateTimeParseException;
import java.util.Arrays;
import java.util.Date;
+import org.apache.kylin.common.exception.KylinException;
import org.apache.kylin.junit.annotation.MultiTimezoneTest;
import org.joda.time.DateTime;
import org.joda.time.DateTimeZone;
import org.joda.time.format.DateTimeFormat;
import org.joda.time.format.DateTimeFormatter;
import org.junit.Assert;
+import org.junit.Test;
/**
* Created by dongli on 1/4/16.
@@ -189,4 +191,53 @@ public class DateFormatTest {
}
}
+
+ @Test
+ public void testStringToMillis() {
+ // 2022-12-01 00:00:00
+ long expectedMillis = 1669824000000L;
+
+ Assert.assertEquals(expectedMillis, DateFormat.stringToMillis("2022.12.01 00:00"));
+ Assert.assertEquals(expectedMillis, DateFormat.stringToMillis("202212"));
+ Assert.assertEquals(expectedMillis, DateFormat.stringToMillis("2022.12.01 00:00:00"));
+ Assert.assertEquals(expectedMillis, DateFormat.stringToMillis("2022-12-01 00:00:00:000"));
+ Assert.assertEquals(expectedMillis, DateFormat.stringToMillis("20221201"));
+ Assert.assertEquals(expectedMillis, DateFormat.stringToMillis("2022.12.01 00:00:00:000"));
+ Assert.assertEquals(expectedMillis, DateFormat.stringToMillis("2022.12.01"));
+ Assert.assertEquals(expectedMillis, DateFormat.stringToMillis("2022/12/01 00:00"));
+ Assert.assertEquals(expectedMillis, DateFormat.stringToMillis("2022-12-01 00:00:00.000"));
+ Assert.assertEquals(expectedMillis, DateFormat.stringToMillis("2022/12/01 00:00:00:000"));
+ Assert.assertEquals(expectedMillis, DateFormat.stringToMillis("20221201 00:00"));
+ Assert.assertEquals(expectedMillis, DateFormat.stringToMillis("2022-12-01 00:00"));
+ Assert.assertEquals(expectedMillis, DateFormat.stringToMillis("20221201 00:00:00:000"));
+ Assert.assertEquals(expectedMillis, DateFormat.stringToMillis("20221201 00:00:00"));
+ }
+
+ @Test
+ public void testStringToMillisSupplement() {
+ long expectedMillis = 1669824000000L;
+
+ Assert.assertEquals(expectedMillis, DateFormat.stringToMillis("2022/12/01 00:00:00"));
+ Assert.assertEquals(expectedMillis, DateFormat.stringToMillis("20221201T00:00:00.000Z"));
+ Assert.assertEquals(expectedMillis, DateFormat.stringToMillis("2022-12"));
+ Assert.assertEquals(expectedMillis, DateFormat.stringToMillis("2022.12.01T00:00:00.000Z"));
+ Assert.assertEquals(expectedMillis, DateFormat.stringToMillis("2022-12-01"));
+ Assert.assertEquals(expectedMillis, DateFormat.stringToMillis("2022-12-01 00:00:00"));
+ Assert.assertEquals(expectedMillis, DateFormat.stringToMillis("2022/12/01"));
+ Assert.assertEquals(expectedMillis, DateFormat.stringToMillis("20221201 00:00:00.000"));
+ Assert.assertEquals(expectedMillis, DateFormat.stringToMillis("2022-12-01T00:00:00.000Z"));
+ Assert.assertEquals(expectedMillis, DateFormat.stringToMillis("2022/12/01T00:00:00.000Z"));
+ Assert.assertEquals(expectedMillis, DateFormat.stringToMillis("2022.12.01 00:00:00.000"));
+ Assert.assertEquals(expectedMillis, DateFormat.stringToMillis("2022-12-01T00:00:00.000+08:00"));
+ Assert.assertEquals(expectedMillis, DateFormat.stringToMillis("2022/12/01 00:00:00.000"));
+ Assert.assertEquals(expectedMillis, DateFormat.stringToMillis("1669824000000000"));
+ Assert.assertEquals(expectedMillis, DateFormat.stringToMillis("1669824000000"));
+ Assert.assertEquals(expectedMillis, DateFormat.stringToMillis("1669824000"));
+ }
+
+ @Test
+ public void testUnsupportedStringToMillis() {
+ Assert.assertThrows(KylinException.class, () -> DateFormat.stringToMillis("12/01"));
+ Assert.assertThrows(KylinException.class, () -> DateFormat.stringToMillis("-12345"));
+ }
}
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/measure/MeasureTypeFactory.java b/src/core-metadata/src/main/java/org/apache/kylin/measure/MeasureTypeFactory.java
index ff812fcabf..48344c876e 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/measure/MeasureTypeFactory.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/measure/MeasureTypeFactory.java
@@ -34,6 +34,7 @@ import org.apache.kylin.measure.extendedcolumn.ExtendedColumnMeasureType;
import org.apache.kylin.measure.hllc.HLLCMeasureType;
import org.apache.kylin.measure.percentile.PercentileMeasureType;
import org.apache.kylin.measure.raw.RawMeasureType;
+import org.apache.kylin.measure.sumlc.SumLCMeasureType;
import org.apache.kylin.measure.topn.TopNMeasureType;
import org.apache.kylin.metadata.datatype.DataType;
import org.apache.kylin.metadata.datatype.DataTypeSerializer;
@@ -120,6 +121,7 @@ abstract public class MeasureTypeFactory<T> {
factoryInsts.add(new IntersectMeasureType.Factory());
factoryInsts.add(new CollectSetMeasureType.Factory());
factoryInsts.add(new CorrMeasureType.Factory());
+ factoryInsts.add(new SumLCMeasureType.Factory());
logger.info("Checking custom measure types from kylin config");
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/measure/sumlc/SumLCCounter.java b/src/core-metadata/src/main/java/org/apache/kylin/measure/sumlc/SumLCCounter.java
new file mode 100644
index 0000000000..dfca82230b
--- /dev/null
+++ b/src/core-metadata/src/main/java/org/apache/kylin/measure/sumlc/SumLCCounter.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.measure.sumlc;
+
+import java.io.Serializable;
+import java.math.BigDecimal;
+import java.util.Map;
+import java.util.function.BiFunction;
+
+import com.google.common.collect.Maps;
+
+public class SumLCCounter implements Serializable {
+ private static final Map<String, BiFunction<Number, Number, Number>> MERGE_FUNC_MAP = Maps.newHashMap();
+
+ static {
+ MERGE_FUNC_MAP.put(Long.class.getSimpleName(), (s1, s2) -> Long.sum((Long) s1, (Long) s2));
+ MERGE_FUNC_MAP.put(Double.class.getSimpleName(), (s1, s2) -> Double.sum((Double) s1, (Double) s2));
+ MERGE_FUNC_MAP.put(BigDecimal.class.getSimpleName(), (s1, s2) -> ((BigDecimal) s1).add((BigDecimal) s2));
+ }
+
+ Number sumLC;
+ Long timestamp;
+
+ public SumLCCounter() {
+
+ }
+
+ public SumLCCounter(Number sumLC, Long timestamp) {
+ this.sumLC = numericTypeConversion(sumLC);
+ this.timestamp = timestamp;
+ }
+
+ public static SumLCCounter merge(SumLCCounter current, Number sumLC, Long timestamp) {
+ SumLCCounter merged = new SumLCCounter(sumLC, timestamp);
+ return merge(current, merged);
+ }
+
+ public static SumLCCounter merge(SumLCCounter value1, SumLCCounter value2) {
+ if (value1 == null || value1.timestamp == null)
+ return value2;
+ if (value2 == null || value2.timestamp == null)
+ return value1;
+ if (value2.timestamp > value1.timestamp) {
+ return value2;
+ } else if (value1.timestamp > value2.timestamp) {
+ return value1;
+ } else {
+ return mergeSum(value1, value2);
+ }
+ }
+
+ private static SumLCCounter mergeSum(SumLCCounter cnt1, SumLCCounter cnt2) {
+ if (cnt1.sumLC == null)
+ return cnt2;
+ if (cnt2.sumLC == null)
+ return cnt1;
+ String sumLCTypeName = cnt1.sumLC.getClass().getSimpleName();
+ Number semiSum = MERGE_FUNC_MAP.get(sumLCTypeName).apply(cnt1.sumLC, cnt2.sumLC);
+ return new SumLCCounter(semiSum, cnt1.timestamp);
+ }
+
+ private static Number numericTypeConversion(Number input) {
+ if (input instanceof Byte || input instanceof Short || input instanceof Integer) {
+ return input.longValue();
+ } else if (input instanceof Float) {
+ return input.doubleValue();
+ } else {
+ return input;
+ }
+ }
+
+ public Number getSumLC() {
+ return sumLC;
+ }
+
+ public Long getTimestamp() {
+ return timestamp;
+ }
+
+}
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/measure/sumlc/SumLCMeasureType.java b/src/core-metadata/src/main/java/org/apache/kylin/measure/sumlc/SumLCMeasureType.java
new file mode 100644
index 0000000000..c94c29e54d
--- /dev/null
+++ b/src/core-metadata/src/main/java/org/apache/kylin/measure/sumlc/SumLCMeasureType.java
@@ -0,0 +1,172 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.measure.sumlc;
+
+import java.nio.ByteBuffer;
+import java.util.Map;
+
+import org.apache.kylin.common.util.Dictionary;
+import org.apache.kylin.measure.MeasureAggregator;
+import org.apache.kylin.measure.MeasureIngester;
+import org.apache.kylin.measure.MeasureType;
+import org.apache.kylin.measure.MeasureTypeFactory;
+import org.apache.kylin.metadata.datatype.DataType;
+import org.apache.kylin.metadata.datatype.DataTypeSerializer;
+import org.apache.kylin.metadata.model.MeasureDesc;
+import org.apache.kylin.metadata.model.TblColRef;
+
+import com.google.common.collect.ImmutableMap;
+
+public class SumLCMeasureType extends MeasureType<SumLCCounter> {
+ public static final String FUNC_SUM_LC = "SUM_LC";
+ public static final String DATATYPE_SUM_LC = "sum_lc";
+ static final Map<String, Class<?>> UDAF_MAP = ImmutableMap.of(SumLCMeasureType.FUNC_SUM_LC, SumLCAggFunc.class);
+
+ public SumLCMeasureType(String funcName, DataType dataType) {
+
+ }
+
+ @Override
+ public MeasureIngester<SumLCCounter> newIngester() {
+ return new MeasureIngester<SumLCCounter>() {
+ @Override
+ public SumLCCounter valueOf(String[] values, MeasureDesc measureDesc,
+ Map<TblColRef, Dictionary<String>> dictionaryMap) {
+ return null;
+ }
+ };
+ }
+
+ @Override
+ public MeasureAggregator<SumLCCounter> newAggregator() {
+ return new MeasureAggregator<SumLCCounter>() {
+ @Override
+ public void reset() {
+ // left over issue, default implementation ignored
+ }
+
+ @Override
+ public void aggregate(SumLCCounter value) {
+ // left over issue, default implementation ignored
+ }
+
+ @Override
+ public SumLCCounter aggregate(SumLCCounter value1, SumLCCounter value2) {
+ return null;
+ }
+
+ @Override
+ public SumLCCounter getState() {
+ return null;
+ }
+
+ @Override
+ public int getMemBytesEstimate() {
+ return 0;
+ }
+ };
+ }
+
+ @Override
+ public boolean needRewrite() {
+ return true;
+ }
+
+ @Override
+ public Map<String, Class<?>> getRewriteCalciteAggrFunctions() {
+ return UDAF_MAP;
+ }
+
+ public static class Factory extends MeasureTypeFactory<SumLCCounter> {
+
+ @Override
+ public MeasureType<SumLCCounter> createMeasureType(String funcName, DataType dataType) {
+ return new SumLCMeasureType(funcName, dataType);
+ }
+
+ @Override
+ public String getAggrFunctionName() {
+ return FUNC_SUM_LC;
+ }
+
+ @Override
+ public String getAggrDataTypeName() {
+ return DATATYPE_SUM_LC;
+ }
+
+ @Override
+ public Class<? extends DataTypeSerializer<SumLCCounter>> getAggrDataTypeSerializer() {
+ return SumLCSerializer.class;
+ }
+ }
+
+ /**
+ * This class is used for registering sum_lc to calcite schema, no need to implement the functions
+ */
+ public static class SumLCAggFunc {
+
+ public static SumLCCounter init() {
+ return null;
+ }
+
+ public static SumLCCounter add(SumLCCounter cur, Object v, Object r) {
+ return null;
+ }
+
+ public static SumLCCounter merge(SumLCCounter counter0, SumLCCounter counter1) {
+ return null;
+ }
+
+ public static Object result(SumLCCounter counter) {
+ return null;
+ }
+ }
+
+ public static class SumLCSerializer extends DataTypeSerializer<SumLCCounter> {
+
+ public SumLCSerializer(DataType dataType) {
+
+ }
+
+ @Override
+ public void serialize(SumLCCounter value, ByteBuffer out) {
+ // left over issue, default implementation ignored
+ }
+
+ @Override
+ public SumLCCounter deserialize(ByteBuffer in) {
+ return null;
+ }
+
+ @Override
+ public int peekLength(ByteBuffer in) {
+ return 0;
+ }
+
+ @Override
+ public int maxLength() {
+ return 0;
+ }
+
+ @Override
+ public int getStorageBytesEstimate() {
+ return 0;
+ }
+ }
+}
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DataType.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DataType.java
index f9ef1bc103..ce7081c1bc 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DataType.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DataType.java
@@ -171,9 +171,9 @@ public class DataType implements Serializable {
}
public static DataType getType(String type) {
- if (type == null)
+ if (type == null) {
return null;
-
+ }
DataType dataType = new DataType(type);
DataType cached = CACHE.get(dataType);
if (cached == null) {
@@ -409,10 +409,13 @@ public class DataType implements Serializable {
public static final BytesSerializer<DataType> serializer = new BytesSerializer<DataType>() {
@Override
public void serialize(DataType value, ByteBuffer out) {
+ serializeDataType(value, out);
+ }
+
+ private void serializeDataType(DataType value, ByteBuffer out) {
BytesUtil.writeUTFString(value.name, out);
BytesUtil.writeVInt(value.precision, out);
BytesUtil.writeVInt(value.scale, out);
-
}
@Override
@@ -420,7 +423,6 @@ public class DataType implements Serializable {
String name = BytesUtil.readUTFString(in);
int precision = BytesUtil.readVInt(in);
int scale = BytesUtil.readVInt(in);
-
return new DataType(name, precision, scale);
}
};
diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
index c66ed0ce8e..b2a212f84f 100644
--- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
+++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
@@ -43,6 +43,7 @@ import java.util.stream.Collectors;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang.StringUtils;
+import org.apache.directory.api.util.Strings;
import org.apache.kylin.common.exception.KylinException;
import org.apache.kylin.measure.MeasureType;
import org.apache.kylin.measure.MeasureTypeFactory;
@@ -55,6 +56,7 @@ import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.base.Joiner;
+import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
@@ -91,21 +93,31 @@ public class FunctionDesc implements Serializable {
public static String proposeReturnType(String expression, String colDataType, Map<String, String> override,
boolean saveCheck) {
- String returnType = override.getOrDefault(expression,
- EXPRESSION_DEFAULT_TYPE_MAP.getOrDefault(expression, colDataType));
- if (saveCheck && colDataType != null && DataType.getType(colDataType).isStringFamily()) {
+ if (saveCheck) {
switch (expression) {
case FunctionDesc.FUNC_SUM:
- case FunctionDesc.FUNC_PERCENTILE:
- throw new KylinException(INVALID_MEASURE_DATA_TYPE,
- String.format(Locale.ROOT, "Invalid column type %s for measure %s", colDataType, expression));
+ case FunctionDesc.FUNC_PERCENTILE: {
+ if (colDataType != null && DataType.getType(colDataType).isStringFamily()) {
+ throw new KylinException(INVALID_MEASURE_DATA_TYPE, String.format(Locale.ROOT,
+ "Invalid column type %s for measure %s", colDataType, expression));
+ }
+ break;
+ }
+ case FunctionDesc.FUNC_SUM_LC: {
+ Preconditions.checkArgument(Strings.isNotEmpty(colDataType),
+ "SUM_LC Measure's input type shouldn't be null or empty");
+ checkSumLCDataType(colDataType);
+ break;
+ }
default:
break;
}
}
- switch (expression) {
- case FunctionDesc.FUNC_SUM:
+ String returnType = override.getOrDefault(expression,
+ EXPRESSION_DEFAULT_TYPE_MAP.getOrDefault(expression, colDataType));
+ // widen return type for sum or sum_lc measure
+ if (FunctionDesc.FUNC_SUM.equals(expression) || FunctionDesc.FUNC_SUM_LC.equals(expression)) {
if (colDataType != null) {
DataType type = DataType.getType(returnType);
if (type.isIntegerFamily()) {
@@ -119,13 +131,19 @@ public class FunctionDesc implements Serializable {
} else {
returnType = "decimal(19,4)";
}
- break;
- default:
- break;
}
return returnType;
}
+ private static void checkSumLCDataType(String dataTypeName) {
+ DataType dataType = DataType.getType(dataTypeName);
+ if (!dataType.isNumberFamily()) {
+ throw new KylinException(INVALID_MEASURE_DATA_TYPE,
+ String.format(Locale.ROOT, "SUM_LC Measure's return type '%s' is illegal. It must be one of %s",
+ dataType, DataType.NUMBER_FAMILY));
+ }
+ }
+
public static final String FUNC_SUM = "SUM";
public static final String FUNC_MIN = "MIN";
public static final String FUNC_MAX = "MAX";
@@ -147,6 +165,7 @@ public class FunctionDesc implements Serializable {
public static final String FUNC_PERCENTILE = "PERCENTILE_APPROX";
public static final String FUNC_GROUPING = "GROUPING";
public static final String FUNC_TOP_N = "TOP_N";
+ public static final String FUNC_SUM_LC = "SUM_LC";
public static final ImmutableSet<String> DIMENSION_AS_MEASURES = ImmutableSet.<String> builder()
.add(FUNC_MAX, FUNC_MIN, FUNC_COUNT_DISTINCT).build();
public static final ImmutableSet<String> NOT_SUPPORTED_FUNCTION = ImmutableSet.<String> builder().build();
@@ -195,19 +214,30 @@ public class FunctionDesc implements Serializable {
for (ParameterDesc p : getParameters()) {
if (p.isColumnType()) {
TblColRef colRef = model.findColumn(p.getValue());
- returnDataType = DataType.getType(
- proposeReturnType(expression, colRef.getDatatype(), Maps.newHashMap(), model.isSaveCheck()));
p.setValue(colRef.getIdentity());
p.setColRef(colRef);
+ if (expression.equals(FUNC_SUM_LC)) {
+ if (Objects.isNull(returnDataType)) {
+ // use the first column to init returnType and returnDataType, ignore the second timestamp column
+ returnType = proposeReturnType(expression, colRef.getDatatype(), Maps.newHashMap(),
+ model.isSaveCheck());
+ returnDataType = DataType.getType(returnType);
+ }
+ } else {
+ returnDataType = DataType.getType(proposeReturnType(expression, colRef.getDatatype(),
+ Maps.newHashMap(), model.isSaveCheck()));
+ }
}
}
- if (returnDataType == null) {
- returnDataType = DataType.getType(BIGINT);
- }
- if (!StringUtils.isEmpty(returnType)) {
- returnDataType = DataType.getType(returnType);
+ if (!expression.equals(FUNC_SUM_LC)) {
+ if (returnDataType == null) {
+ returnDataType = DataType.getType(BIGINT);
+ }
+ if (!StringUtils.isEmpty(returnType)) {
+ returnDataType = DataType.getType(returnType);
+ }
+ returnType = returnDataType.toString();
}
- returnType = returnDataType.toString();
}
private void reInitMeasureType() {
diff --git a/src/core-metadata/src/test/java/org/apache/kylin/metadata/model/util/FunctionDescTest.java b/src/core-metadata/src/test/java/org/apache/kylin/metadata/model/util/FunctionDescTest.java
index 226b8ac3b7..77ed4f6ecd 100644
--- a/src/core-metadata/src/test/java/org/apache/kylin/metadata/model/util/FunctionDescTest.java
+++ b/src/core-metadata/src/test/java/org/apache/kylin/metadata/model/util/FunctionDescTest.java
@@ -26,14 +26,14 @@ import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.apache.kylin.common.exception.KylinException;
+import org.apache.kylin.common.util.NLocalFileMetadataTestCase;
import org.apache.kylin.metadata.datatype.DataType;
import org.apache.kylin.metadata.model.FunctionDesc;
+import org.apache.kylin.metadata.model.NDataModel;
+import org.apache.kylin.metadata.model.NDataModelManager;
import org.apache.kylin.metadata.model.ParameterDesc;
import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.common.util.NLocalFileMetadataTestCase;
-import org.apache.kylin.metadata.model.NDataModel;
-import org.apache.kylin.metadata.model.NDataModelManager;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
@@ -86,6 +86,12 @@ public class FunctionDescTest extends NLocalFileMetadataTestCase {
Assert.fail();
} catch (KylinException ignored) {
}
+ Assert.assertThrows(IllegalArgumentException.class,
+ () -> FunctionDesc.proposeReturnType("SUM_LC", "", Maps.newHashMap(), true));
+ Assert.assertThrows(KylinException.class,
+ () -> FunctionDesc.proposeReturnType("SUM_LC", "char", Maps.newHashMap(), true));
+ String returnType = FunctionDesc.proposeReturnType("SUM_LC", "bigint", Maps.newHashMap(), true);
+ Assert.assertEquals("bigint", returnType);
}
@Test
diff --git a/src/core-metadata/src/test/java/org/apache/kylin/metadata/project/NProjectManagerTest.java b/src/core-metadata/src/test/java/org/apache/kylin/metadata/project/NProjectManagerTest.java
index d87805c140..0d2e6cbaf3 100644
--- a/src/core-metadata/src/test/java/org/apache/kylin/metadata/project/NProjectManagerTest.java
+++ b/src/core-metadata/src/test/java/org/apache/kylin/metadata/project/NProjectManagerTest.java
@@ -26,10 +26,9 @@ import org.apache.hadoop.fs.Path;
import org.apache.kylin.common.KapConfig;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.exception.KylinException;
-import org.apache.kylin.common.util.HadoopUtil;
import org.apache.kylin.common.hystrix.NCircuitBreaker;
+import org.apache.kylin.common.util.HadoopUtil;
import org.apache.kylin.common.util.NLocalFileMetadataTestCase;
-import org.apache.kylin.metadata.project.ProjectInstance;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@@ -69,7 +68,7 @@ public class NProjectManagerTest extends NLocalFileMetadataTestCase {
}
val projects = projectManager.listAllProjects();
- Assert.assertEquals(27, projects.size());
+ Assert.assertEquals(28, projects.size());
Assert.assertTrue(projects.stream().noneMatch(p -> p.getName().equals("test")));
}
diff --git a/src/examples/test_case_data/localmeta/metadata/_global/project/sum_lc.json b/src/examples/test_case_data/localmeta/metadata/_global/project/sum_lc.json
new file mode 100644
index 0000000000..2fdd47af4f
--- /dev/null
+++ b/src/examples/test_case_data/localmeta/metadata/_global/project/sum_lc.json
@@ -0,0 +1,35 @@
+{
+ "uuid" : "d1ceb3a4-4d2c-27af-5b7d-92c8a4055776",
+ "last_modified" : 1667464991353,
+ "create_time" : 1667464991347,
+ "version" : "4.0.0.0",
+ "name" : "sum_lc",
+ "owner" : "ADMIN",
+ "status" : "ENABLED",
+ "create_time_utc" : 1667464991347,
+ "default_database" : "DEFAULT",
+ "description" : "",
+ "principal" : null,
+ "keytab" : null,
+ "maintain_model_type" : "MANUAL_MAINTAIN",
+ "override_kylin_properties" : {
+ "kylin.metadata.semi-automatic-mode" : "false",
+ "kylin.query.metadata.expose-computed-column" : "true",
+ "kylin.source.default" : "9"
+ },
+ "segment_config" : {
+ "auto_merge_enabled" : false,
+ "auto_merge_time_ranges" : [ "WEEK", "MONTH", "QUARTER", "YEAR" ],
+ "volatile_range" : {
+ "volatile_range_number" : 0,
+ "volatile_range_enabled" : false,
+ "volatile_range_type" : "DAY"
+ },
+ "retention_range" : {
+ "retention_range_number" : 1,
+ "retention_range_enabled" : false,
+ "retention_range_type" : "MONTH"
+ },
+ "create_empty_segment_enabled" : false
+ }
+}
\ No newline at end of file
diff --git a/src/examples/test_case_data/localmeta/metadata/sum_lc/dataflow/f35f2937-9e4d-347a-7465-d64df939e7d6.json b/src/examples/test_case_data/localmeta/metadata/sum_lc/dataflow/f35f2937-9e4d-347a-7465-d64df939e7d6.json
new file mode 100644
index 0000000000..f5f39f4889
--- /dev/null
+++ b/src/examples/test_case_data/localmeta/metadata/sum_lc/dataflow/f35f2937-9e4d-347a-7465-d64df939e7d6.json
@@ -0,0 +1,13 @@
+{
+ "uuid" : "f35f2937-9e4d-347a-7465-d64df939e7d6",
+ "last_modified" : 1667467391143,
+ "create_time" : 1667465241650,
+ "version" : "4.0.0.0",
+ "status" : "ONLINE",
+ "last_status" : null,
+ "cost" : 50,
+ "query_hit_count" : 0,
+ "last_query_time" : 0,
+ "layout_query_hit_count" : { },
+ "segments" : [ ]
+}
\ No newline at end of file
diff --git a/src/examples/test_case_data/localmeta/metadata/sum_lc/index_plan/f35f2937-9e4d-347a-7465-d64df939e7d6.json b/src/examples/test_case_data/localmeta/metadata/sum_lc/index_plan/f35f2937-9e4d-347a-7465-d64df939e7d6.json
new file mode 100644
index 0000000000..9b29b1fd06
--- /dev/null
+++ b/src/examples/test_case_data/localmeta/metadata/sum_lc/index_plan/f35f2937-9e4d-347a-7465-d64df939e7d6.json
@@ -0,0 +1,43 @@
+{
+ "uuid" : "f35f2937-9e4d-347a-7465-d64df939e7d6",
+ "last_modified" : 1667467391117,
+ "create_time" : 1667293061109,
+ "version" : "4.0.0.0",
+ "description" : null,
+ "rule_based_index" : {
+ "dimensions" : [ 1, 6 ],
+ "measures" : [ 100000, 100001, 100002, 100003, 100004, 100005, 100006, 100007, 100008 ],
+ "global_dim_cap" : null,
+ "aggregation_groups" : [ {
+ "includes" : [ 1, 6 ],
+ "measures" : [ 100000, 100001, 100002, 100003, 100004, 100005, 100006, 100007, 100008 ],
+ "select_rule" : {
+ "hierarchy_dims" : [ ],
+ "mandatory_dims" : [ ],
+ "joint_dims" : [ ]
+ },
+ "index_range" : "EMPTY"
+ } ],
+ "layout_id_mapping" : [ 30001, 40001, 50001 ],
+ "parent_forward" : 3,
+ "index_start_id" : 30000,
+ "last_modify_time" : 1667370467484,
+ "layout_black_list" : [ ],
+ "scheduler_version" : 2,
+ "index_update_enabled" : true,
+ "base_layout_enabled" : true
+ },
+ "indexes" : [ ],
+ "override_properties" : { },
+ "to_be_deleted_indexes" : [ ],
+ "auto_merge_time_ranges" : null,
+ "retention_range" : 0,
+ "engine_type" : 80,
+ "next_aggregation_index_id" : 60000,
+ "next_table_index_id" : 20000000000,
+ "agg_shard_by_columns" : [ ],
+ "extend_partition_columns" : [ ],
+ "layout_bucket_num" : { },
+ "approved_additional_recs" : 0,
+ "approved_removal_recs" : 0
+}
\ No newline at end of file
diff --git a/src/examples/test_case_data/localmeta/metadata/sum_lc/model_desc/f35f2937-9e4d-347a-7465-d64df939e7d6.json b/src/examples/test_case_data/localmeta/metadata/sum_lc/model_desc/f35f2937-9e4d-347a-7465-d64df939e7d6.json
new file mode 100644
index 0000000000..ddc2b999de
--- /dev/null
+++ b/src/examples/test_case_data/localmeta/metadata/sum_lc/model_desc/f35f2937-9e4d-347a-7465-d64df939e7d6.json
@@ -0,0 +1,267 @@
+{
+ "uuid" : "f35f2937-9e4d-347a-7465-d64df939e7d6",
+ "last_modified" : 1667465241644,
+ "create_time" : 1667293060578,
+ "version" : "4.0.0.0",
+ "alias" : "sum_lc_multi_data_type_test",
+ "owner" : "ADMIN",
+ "config_last_modifier" : null,
+ "config_last_modified" : 0,
+ "description" : null,
+ "fact_table" : "SSB.SUMLC_EXTEND_4X",
+ "fact_table_alias" : null,
+ "management_type" : "MODEL_BASED",
+ "join_tables" : [ ],
+ "filter_condition" : "",
+ "partition_desc" : {
+ "partition_date_column" : "SUMLC_EXTEND_4X.TX_DATE",
+ "partition_date_start" : 0,
+ "partition_date_format" : "yyyy-MM-dd",
+ "partition_type" : "APPEND",
+ "partition_condition_builder" : "org.apache.kylin.metadata.model.PartitionDesc$DefaultPartitionConditionBuilder"
+ },
+ "capacity" : "MEDIUM",
+ "segment_config" : {
+ "auto_merge_enabled" : null,
+ "auto_merge_time_ranges" : null,
+ "volatile_range" : null,
+ "retention_range" : null,
+ "create_empty_segment_enabled" : false
+ },
+ "data_check_desc" : null,
+ "semantic_version" : 0,
+ "storage_type" : 0,
+ "model_type" : "BATCH",
+ "all_named_columns" : [ {
+ "id" : 0,
+ "name" : "INT_DATA",
+ "column" : "SUMLC_EXTEND_4X.INT_DATA"
+ }, {
+ "id" : 1,
+ "name" : "ACCOUNT",
+ "column" : "SUMLC_EXTEND_4X.ACCOUNT",
+ "status" : "DIMENSION"
+ }, {
+ "id" : 2,
+ "name" : "TINYINT_DATA",
+ "column" : "SUMLC_EXTEND_4X.TINYINT_DATA"
+ }, {
+ "id" : 3,
+ "name" : "DOUBLE_DATA",
+ "column" : "SUMLC_EXTEND_4X.DOUBLE_DATA"
+ }, {
+ "id" : 4,
+ "name" : "SMALLINT_DATA",
+ "column" : "SUMLC_EXTEND_4X.SMALLINT_DATA"
+ }, {
+ "id" : 5,
+ "name" : "DECIMAL_DATA",
+ "column" : "SUMLC_EXTEND_4X.DECIMAL_DATA"
+ }, {
+ "id" : 6,
+ "name" : "TX_DATE",
+ "column" : "SUMLC_EXTEND_4X.TX_DATE",
+ "status" : "DIMENSION"
+ }, {
+ "id" : 7,
+ "name" : "MILLIS",
+ "column" : "SUMLC_EXTEND_4X.MILLIS"
+ }, {
+ "id" : 8,
+ "name" : "FLOAT_DATA",
+ "column" : "SUMLC_EXTEND_4X.FLOAT_DATA"
+ }, {
+ "id" : 9,
+ "name" : "BIGINT_DATA",
+ "column" : "SUMLC_EXTEND_4X.BIGINT_DATA"
+ }, {
+ "id" : 10,
+ "name" : "TINYINT_CC",
+ "column" : "SUMLC_EXTEND_4X.TINYINT_CC"
+ } ],
+ "all_measures" : [ {
+ "name" : "COUNT_ALL",
+ "function" : {
+ "expression" : "COUNT",
+ "parameters" : [ {
+ "type" : "constant",
+ "value" : "1"
+ } ],
+ "returntype" : "bigint"
+ },
+ "column" : null,
+ "comment" : null,
+ "id" : 100000,
+ "type" : "NORMAL",
+ "internal_ids" : [ ]
+ }, {
+ "name" : "sumlc_decimal_tx_date",
+ "function" : {
+ "expression" : "SUM_LC",
+ "parameters" : [ {
+ "type" : "column",
+ "value" : "SUMLC_EXTEND_4X.DECIMAL_DATA"
+ }, {
+ "type" : "column",
+ "value" : "SUMLC_EXTEND_4X.TX_DATE"
+ } ],
+ "returntype" : "decimal(19,7)"
+ },
+ "column" : null,
+ "comment" : "",
+ "id" : 100001,
+ "type" : "NORMAL",
+ "internal_ids" : [ ]
+ }, {
+ "name" : "sumlc_tinyint_tx_date",
+ "function" : {
+ "expression" : "SUM_LC",
+ "parameters" : [ {
+ "type" : "column",
+ "value" : "SUMLC_EXTEND_4X.TINYINT_DATA"
+ }, {
+ "type" : "column",
+ "value" : "SUMLC_EXTEND_4X.TX_DATE"
+ } ],
+ "returntype" : "bigint"
+ },
+ "column" : null,
+ "comment" : "",
+ "id" : 100002,
+ "type" : "NORMAL",
+ "internal_ids" : [ ]
+ }, {
+ "name" : "sumlc_smallint_tx_date",
+ "function" : {
+ "expression" : "SUM_LC",
+ "parameters" : [ {
+ "type" : "column",
+ "value" : "SUMLC_EXTEND_4X.SMALLINT_DATA"
+ }, {
+ "type" : "column",
+ "value" : "SUMLC_EXTEND_4X.TX_DATE"
+ } ],
+ "returntype" : "bigint"
+ },
+ "column" : null,
+ "comment" : "",
+ "id" : 100003,
+ "type" : "NORMAL",
+ "internal_ids" : [ ]
+ }, {
+ "name" : "sumlc_int_tx_date",
+ "function" : {
+ "expression" : "SUM_LC",
+ "parameters" : [ {
+ "type" : "column",
+ "value" : "SUMLC_EXTEND_4X.INT_DATA"
+ }, {
+ "type" : "column",
+ "value" : "SUMLC_EXTEND_4X.TX_DATE"
+ } ],
+ "returntype" : "bigint"
+ },
+ "column" : null,
+ "comment" : "",
+ "id" : 100004,
+ "type" : "NORMAL",
+ "internal_ids" : [ ]
+ }, {
+ "name" : "sumlc_bigint_tx_date",
+ "function" : {
+ "expression" : "SUM_LC",
+ "parameters" : [ {
+ "type" : "column",
+ "value" : "SUMLC_EXTEND_4X.BIGINT_DATA"
+ }, {
+ "type" : "column",
+ "value" : "SUMLC_EXTEND_4X.TX_DATE"
+ } ],
+ "returntype" : "bigint"
+ },
+ "column" : null,
+ "comment" : "",
+ "id" : 100005,
+ "type" : "NORMAL",
+ "internal_ids" : [ ]
+ }, {
+ "name" : "sumlc_float_tx_date",
+ "function" : {
+ "expression" : "SUM_LC",
+ "parameters" : [ {
+ "type" : "column",
+ "value" : "SUMLC_EXTEND_4X.FLOAT_DATA"
+ }, {
+ "type" : "column",
+ "value" : "SUMLC_EXTEND_4X.TX_DATE"
+ } ],
+ "returntype" : "double"
+ },
+ "column" : null,
+ "comment" : "",
+ "id" : 100006,
+ "type" : "NORMAL",
+ "internal_ids" : [ ]
+ }, {
+ "name" : "sumlc_double_tx_date",
+ "function" : {
+ "expression" : "SUM_LC",
+ "parameters" : [ {
+ "type" : "column",
+ "value" : "SUMLC_EXTEND_4X.DOUBLE_DATA"
+ }, {
+ "type" : "column",
+ "value" : "SUMLC_EXTEND_4X.TX_DATE"
+ } ],
+ "returntype" : "double"
+ },
+ "column" : null,
+ "comment" : "",
+ "id" : 100007,
+ "type" : "NORMAL",
+ "internal_ids" : [ ]
+ }, {
+ "name" : "sumlc_tinyint_cc_tx_date",
+ "function" : {
+ "expression" : "SUM_LC",
+ "parameters" : [ {
+ "type" : "column",
+ "value" : "SUMLC_EXTEND_4X.TINYINT_CC"
+ }, {
+ "type" : "column",
+ "value" : "SUMLC_EXTEND_4X.TX_DATE"
+ } ],
+ "returntype" : "bigint"
+ },
+ "column" : null,
+ "comment" : "",
+ "id" : 100008,
+ "type" : "NORMAL",
+ "internal_ids" : [ ]
+ } ],
+ "recommendations_count" : 0,
+ "computed_columns" : [ {
+ "tableIdentity" : "SSB.SUMLC_EXTEND_4X",
+ "tableAlias" : "SUMLC_EXTEND_4X",
+ "columnName" : "TINYINT_CC",
+ "expression" : "SUMLC_EXTEND_4X.TINYINT_DATA * 2",
+ "innerExpression" : "`SUMLC_EXTEND_4X`.`TINYINT_DATA` * 2",
+ "datatype" : "INTEGER",
+ "comment" : null,
+ "rec_uuid" : null
+ } ],
+ "canvas" : {
+ "coordinate" : {
+ "SUMLC_EXTEND_4X" : {
+ "x" : 307.9999966091586,
+ "y" : 27.444441053602443,
+ "width" : 200.0,
+ "height" : 522.2222222222221
+ }
+ },
+ "zoom" : 9.0
+ },
+ "multi_partition_desc" : null,
+ "multi_partition_key_mapping" : null,
+ "fusion_id" : null
+}
\ No newline at end of file
diff --git a/src/examples/test_case_data/localmeta/metadata/sum_lc/table/SSB.SUMLC_EXTEND_4X.json b/src/examples/test_case_data/localmeta/metadata/sum_lc/table/SSB.SUMLC_EXTEND_4X.json
new file mode 100644
index 0000000000..85880c9a43
--- /dev/null
+++ b/src/examples/test_case_data/localmeta/metadata/sum_lc/table/SSB.SUMLC_EXTEND_4X.json
@@ -0,0 +1,78 @@
+{
+ "uuid" : "5c5334e5-44ec-d8a3-0357-a63d84b56b06",
+ "last_modified" : 0,
+ "create_time" : 1667465028069,
+ "version" : "4.0.0.0",
+ "name" : "SUMLC_EXTEND_4X",
+ "columns" : [ {
+ "id" : "1",
+ "name" : "TX_DATE",
+ "datatype" : "date",
+ "case_sensitive_name" : "tx_date"
+ }, {
+ "id" : "2",
+ "name" : "MILLIS",
+ "datatype" : "bigint",
+ "case_sensitive_name" : "millis"
+ }, {
+ "id" : "3",
+ "name" : "ACCOUNT",
+ "datatype" : "varchar(4096)",
+ "case_sensitive_name" : "account"
+ }, {
+ "id" : "4",
+ "name" : "TINYINT_DATA",
+ "datatype" : "tinyint",
+ "case_sensitive_name" : "tinyint_data"
+ }, {
+ "id" : "5",
+ "name" : "SMALLINT_DATA",
+ "datatype" : "smallint",
+ "case_sensitive_name" : "smallint_data"
+ }, {
+ "id" : "6",
+ "name" : "INT_DATA",
+ "datatype" : "integer",
+ "case_sensitive_name" : "int_data"
+ }, {
+ "id" : "7",
+ "name" : "BIGINT_DATA",
+ "datatype" : "bigint",
+ "case_sensitive_name" : "bigint_data"
+ }, {
+ "id" : "8",
+ "name" : "FLOAT_DATA",
+ "datatype" : "double",
+ "case_sensitive_name" : "float_data"
+ }, {
+ "id" : "9",
+ "name" : "DOUBLE_DATA",
+ "datatype" : "double",
+ "case_sensitive_name" : "double_data"
+ }, {
+ "id" : "10",
+ "name" : "DECIMAL_DATA",
+ "datatype" : "decimal(9,7)",
+ "case_sensitive_name" : "decimal_data"
+ } ],
+ "source_type" : 9,
+ "table_type" : "MANAGED",
+ "top" : false,
+ "increment_loading" : false,
+ "last_snapshot_path" : null,
+ "last_snapshot_size" : 0,
+ "snapshot_last_modified" : 0,
+ "query_hit_count" : 0,
+ "partition_column" : null,
+ "snapshot_partitions" : { },
+ "snapshot_partitions_info" : { },
+ "snapshot_total_rows" : 0,
+ "snapshot_partition_col" : null,
+ "selected_snapshot_partition_col" : null,
+ "temp_snapshot_path" : null,
+ "snapshot_has_broken" : false,
+ "database" : "SSB",
+ "transactional" : false,
+ "rangePartition" : false,
+ "partition_desc" : null
+}
\ No newline at end of file
diff --git a/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ProjectServiceTest.java b/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ProjectServiceTest.java
index 61720f727c..34160d76c6 100644
--- a/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ProjectServiceTest.java
+++ b/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ProjectServiceTest.java
@@ -56,6 +56,7 @@ import org.apache.kylin.metadata.model.NDataModelManager;
import org.apache.kylin.metadata.project.NProjectManager;
import org.apache.kylin.metadata.project.ProjectInstance;
import org.apache.kylin.metadata.realization.RealizationStatusEnum;
+import org.apache.kylin.metadata.recommendation.candidate.JdbcRawRecStore;
import org.apache.kylin.query.pushdown.PushDownRunnerSparkImpl;
import org.apache.kylin.rest.constant.Constant;
import org.apache.kylin.rest.request.GarbageCleanUpConfigRequest;
@@ -97,7 +98,6 @@ import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import io.kyligence.kap.clickhouse.MockSecondStorage;
-import org.apache.kylin.metadata.recommendation.candidate.JdbcRawRecStore;
import lombok.val;
import lombok.var;
import lombok.extern.slf4j.Slf4j;
@@ -235,14 +235,14 @@ public class ProjectServiceTest extends NLocalFileMetadataTestCase {
public void testGetReadableProjects() {
Mockito.doReturn(true).when(aclEvaluate).hasProjectAdminPermission(Mockito.any(ProjectInstance.class));
List<ProjectInstance> projectInstances = projectService.getReadableProjects("", false);
- Assert.assertEquals(27, projectInstances.size());
+ Assert.assertEquals(28, projectInstances.size());
}
@Test
public void testGetAdminProjects() throws Exception {
Mockito.doReturn(true).when(aclEvaluate).hasProjectAdminPermission(Mockito.any(ProjectInstance.class));
List<ProjectInstance> projectInstances = projectService.getAdminProjects();
- Assert.assertEquals(27, projectInstances.size());
+ Assert.assertEquals(28, projectInstances.size());
}
@Test
@@ -256,7 +256,7 @@ public class ProjectServiceTest extends NLocalFileMetadataTestCase {
public void testGetReadableProjectsHasNoPermissionProject() {
Mockito.doReturn(true).when(aclEvaluate).hasProjectAdminPermission(Mockito.any(ProjectInstance.class));
List<ProjectInstance> projectInstances = projectService.getReadableProjects("", false);
- Assert.assertEquals(27, projectInstances.size());
+ Assert.assertEquals(28, projectInstances.size());
}
diff --git a/src/query-common/src/main/java/org/apache/kylin/query/relnode/KapAggregateRel.java b/src/query-common/src/main/java/org/apache/kylin/query/relnode/KapAggregateRel.java
index bb004b174d..73c50d3d64 100644
--- a/src/query-common/src/main/java/org/apache/kylin/query/relnode/KapAggregateRel.java
+++ b/src/query-common/src/main/java/org/apache/kylin/query/relnode/KapAggregateRel.java
@@ -46,13 +46,13 @@ import org.apache.kylin.common.KapConfig;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.QueryContext;
import org.apache.kylin.measure.corr.CorrMeasureType;
-import org.apache.kylin.metadata.model.FunctionDesc;
-import org.apache.kylin.metadata.model.PartitionDesc;
-import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.cube.cuboid.NLayoutCandidate;
import org.apache.kylin.metadata.cube.model.NDataflow;
+import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.MultiPartitionDesc;
import org.apache.kylin.metadata.model.NDataModel;
+import org.apache.kylin.metadata.model.PartitionDesc;
+import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.query.util.ICutContextStrategy;
import com.google.common.collect.ImmutableList;
@@ -65,7 +65,7 @@ import com.google.common.collect.Sets;
public class KapAggregateRel extends OLAPAggregateRel implements KapRel {
protected static final List<String> supportedFunction = Lists.newArrayList("SUM", "MIN", "MAX", "COUNT_DISTINCT",
- "BITMAP_UUID", "PERCENTILE_APPROX", FunctionDesc.FUNC_BITMAP_BUILD);
+ "BITMAP_UUID", "PERCENTILE_APPROX", FunctionDesc.FUNC_BITMAP_BUILD, FunctionDesc.FUNC_SUM_LC);
private ImmutableList<Integer> rewriteGroupKeys; // preserve the ordering of group keys after CC replacement
private List<ImmutableBitSet> rewriteGroupSets; // group sets with cc replaced
List<AggregateCall> aggregateCalls;
@@ -203,8 +203,8 @@ public class KapAggregateRel extends OLAPAggregateRel implements KapRel {
TblColRef originalColumn = inputColumnRowType.getColumnByIndex(i);
if (null != this.context && this.context.getGroupCCColRewriteMapping().containsKey(originalColumn)) {
groups.add(this.context.getGroupCCColRewriteMapping().get(originalColumn));
- groupKeys
- .add(inputColumnRowType.getIndexByName(this.context.getGroupCCColRewriteMapping().get(originalColumn).getName()));
+ groupKeys.add(inputColumnRowType
+ .getIndexByName(this.context.getGroupCCColRewriteMapping().get(originalColumn).getName()));
} else {
Set<TblColRef> sourceColumns = inputColumnRowType.getSourceColumnsByIndex(i);
groups.addAll(sourceColumns);
@@ -307,7 +307,6 @@ public class KapAggregateRel extends OLAPAggregateRel implements KapRel {
// rebuild rowType & columnRowType
this.rowType = this.deriveRowType();
this.columnRowType = this.buildColumnRowType();
-
}
private Boolean isExactlyMatched() {
@@ -330,7 +329,8 @@ public class KapAggregateRel extends OLAPAggregateRel implements KapRel {
return false;
}
- if (!checkAggCall()) return false;
+ if (!checkAggCall())
+ return false;
Set<String> cuboidDimSet = new HashSet<>();
if (getContext() != null && getContext().storageContext.getCandidate() != null) {
cuboidDimSet = getContext().storageContext.getCandidate().getLayoutEntity().getOrderedDimensions().values()
@@ -403,7 +403,7 @@ public class KapAggregateRel extends OLAPAggregateRel implements KapRel {
private boolean isDimExactlyMatch(Set<String> groupByCols, Set<String> cuboidDimSet) {
return groupByCols.equals(cuboidDimSet) && isSimpleGroupType()
&& (this.context.getInnerGroupByColumns().isEmpty()
- || !this.context.getGroupCCColRewriteMapping().isEmpty());
+ || !this.context.getGroupCCColRewriteMapping().isEmpty());
}
@@ -502,7 +502,8 @@ public class KapAggregateRel extends OLAPAggregateRel implements KapRel {
}
public boolean isContainCountDistinct() {
- return aggregateCalls.stream().anyMatch(agg -> agg.getAggregation().getKind() == SqlKind.COUNT && agg.isDistinct());
+ return aggregateCalls.stream()
+ .anyMatch(agg -> agg.getAggregation().getKind() == SqlKind.COUNT && agg.isDistinct());
}
}
diff --git a/src/query-common/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java b/src/query-common/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
index e944266a82..d68329c0a0 100644
--- a/src/query-common/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
+++ b/src/query-common/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
@@ -101,6 +101,7 @@ public class OLAPAggregateRel extends Aggregate implements OLAPRel {
protected List<AggregateCall> rewriteAggCalls;
protected List<TblColRef> groups;
protected List<FunctionDesc> aggregations;
+
public OLAPAggregateRel(RelOptCluster cluster, RelTraitSet traits, RelNode child, boolean indicator,
ImmutableBitSet groupSet, List<ImmutableBitSet> groupSets, List<AggregateCall> aggCalls)
throws InvalidRelException {
@@ -501,7 +502,6 @@ public class OLAPAggregateRel extends Aggregate implements OLAPRel {
// rebuild aggregate call
return new AggregateCall(newAgg, false, newArgList, fieldType, callName);
-
}
/**
diff --git a/src/query-service/src/test/java/org/apache/kylin/rest/service/QueryHistoryServiceTest.java b/src/query-service/src/test/java/org/apache/kylin/rest/service/QueryHistoryServiceTest.java
index a53f9af8e7..c1f23bb5d7 100644
--- a/src/query-service/src/test/java/org/apache/kylin/rest/service/QueryHistoryServiceTest.java
+++ b/src/query-service/src/test/java/org/apache/kylin/rest/service/QueryHistoryServiceTest.java
@@ -365,7 +365,7 @@ public class QueryHistoryServiceTest extends NLocalFileMetadataTestCase {
// get all tables
tableMap = queryHistoryService.getQueryHistoryTableMap(null);
- Assert.assertEquals(27, tableMap.size());
+ Assert.assertEquals(28, tableMap.size());
// not existing project
try {
diff --git a/src/query/src/main/java/org/apache/kylin/query/engine/QueryRoutingEngine.java b/src/query/src/main/java/org/apache/kylin/query/engine/QueryRoutingEngine.java
index fe1d092ee3..81439609e9 100644
--- a/src/query/src/main/java/org/apache/kylin/query/engine/QueryRoutingEngine.java
+++ b/src/query/src/main/java/org/apache/kylin/query/engine/QueryRoutingEngine.java
@@ -50,6 +50,7 @@ import org.apache.kylin.metadata.query.StructField;
import org.apache.kylin.metadata.querymeta.SelectedColumnMeta;
import org.apache.kylin.metadata.realization.NoStreamingRealizationFoundException;
import org.apache.kylin.query.engine.data.QueryResult;
+import org.apache.kylin.query.exception.NotSupportedSQLException;
import org.apache.kylin.query.mask.QueryResultMasks;
import org.apache.kylin.query.relnode.OLAPContext;
import org.apache.kylin.query.util.PushDownUtil;
@@ -81,6 +82,7 @@ public class QueryRoutingEngine {
queryParams.setDefaultSchema(queryExec.getDefaultSchemaName());
if (queryParams.isForcedToPushDown()) {
+ checkContainsSumLC(queryParams, null);
return pushDownQuery(null, queryParams);
}
@@ -128,6 +130,7 @@ public class QueryRoutingEngine {
if (cause instanceof SQLException && cause.getCause() instanceof KylinException) {
throw (SQLException) cause;
}
+ checkContainsSumLC(queryParams, e);
if (shouldPushdown(cause, queryParams)) {
return pushDownQuery((SQLException) cause, queryParams);
} else {
@@ -146,6 +149,7 @@ public class QueryRoutingEngine {
}
}
}
+ checkContainsSumLC(queryParams, e);
if (shouldPushdown(e, queryParams)) {
return pushDownQuery(e, queryParams);
} else {
@@ -167,6 +171,17 @@ public class QueryRoutingEngine {
return false;
}
+ private void checkContainsSumLC(QueryParams queryParams, Throwable t) {
+ if (queryParams.getSql().contains("sum_lc")) {
+ String message = "There is no aggregate index to answer this query, sum_lc() function now is not supported by other query engine";
+ if (t != null) {
+ throw new NotSupportedSQLException(message, t);
+ } else {
+ throw new NotSupportedSQLException(message);
+ }
+ }
+ }
+
private boolean shouldPushdown(Throwable e, QueryParams queryParams) {
if (queryParams.isForcedToIndex()) {
return false;
diff --git a/src/query/src/test/java/org/apache/kylin/query/engine/AsyncQueryJobTest.java b/src/query/src/test/java/org/apache/kylin/query/engine/AsyncQueryJobTest.java
index 9a1725f1bc..93fe27494f 100644
--- a/src/query/src/test/java/org/apache/kylin/query/engine/AsyncQueryJobTest.java
+++ b/src/query/src/test/java/org/apache/kylin/query/engine/AsyncQueryJobTest.java
@@ -17,11 +17,11 @@
*/
package org.apache.kylin.query.engine;
-import static org.apache.kylin.query.util.AsyncQueryUtil.ASYNC_QUERY_JOB_ID_PRE;
import static org.apache.kylin.metadata.cube.model.NBatchConstants.P_DIST_META_URL;
import static org.apache.kylin.metadata.cube.model.NBatchConstants.P_JOB_ID;
import static org.apache.kylin.metadata.cube.model.NBatchConstants.P_QUERY_CONTEXT;
import static org.apache.kylin.metadata.cube.model.NBatchConstants.P_QUERY_PARAMS;
+import static org.apache.kylin.query.util.AsyncQueryUtil.ASYNC_QUERY_JOB_ID_PRE;
import java.io.BufferedReader;
import java.io.IOException;
@@ -45,11 +45,11 @@ import org.apache.kylin.common.persistence.RawResource;
import org.apache.kylin.common.util.CliCommandExecutor;
import org.apache.kylin.common.util.HadoopUtil;
import org.apache.kylin.common.util.JsonUtil;
+import org.apache.kylin.common.util.NLocalFileMetadataTestCase;
import org.apache.kylin.common.util.ShellException;
import org.apache.kylin.job.exception.ExecuteException;
import org.apache.kylin.job.execution.ExecuteResult;
import org.apache.kylin.query.util.QueryParams;
-import org.apache.kylin.common.util.NLocalFileMetadataTestCase;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@@ -217,7 +217,7 @@ public class AsyncQueryJobTest extends NLocalFileMetadataTestCase {
rawResourceMap.put(zipEntry.getName(), raw);
}
}
- Assert.assertEquals(83, rawResourceMap.size());
+ Assert.assertEquals(84, rawResourceMap.size());
}
private void testKylinConfig(FileSystem workingFileSystem, FileStatus metaFileStatus) throws IOException {
diff --git a/src/query/src/test/java/org/apache/kylin/query/engine/QueryRoutingEngineTest.java b/src/query/src/test/java/org/apache/kylin/query/engine/QueryRoutingEngineTest.java
index f929d50a39..7599095bf6 100644
--- a/src/query/src/test/java/org/apache/kylin/query/engine/QueryRoutingEngineTest.java
+++ b/src/query/src/test/java/org/apache/kylin/query/engine/QueryRoutingEngineTest.java
@@ -39,6 +39,7 @@ import org.apache.kylin.common.util.NLocalFileMetadataTestCase;
import org.apache.kylin.metadata.realization.NoStreamingRealizationFoundException;
import org.apache.kylin.query.QueryExtension;
import org.apache.kylin.query.engine.data.QueryResult;
+import org.apache.kylin.query.exception.NotSupportedSQLException;
import org.apache.kylin.query.util.QueryParams;
import org.apache.kylin.source.adhocquery.PushdownResult;
import org.apache.spark.SparkException;
@@ -304,4 +305,19 @@ public class QueryRoutingEngineTest extends NLocalFileMetadataTestCase {
QueryResult queryResult = queryRoutingEngine.queryWithSqlMassage(queryParams);
Assert.assertEquals(0, queryResult.getSize());
}
+
+ @Test
+ public void testQueryPushDownWithSumLC() {
+ final String sql = "select sum_lc(column, dateColumn) from success_table_2";
+ final String project = "default";
+ KylinConfig kylinconfig = KylinConfig.getInstanceFromEnv();
+ QueryParams queryParams = new QueryParams();
+ queryParams.setProject(project);
+ queryParams.setSql(sql);
+ queryParams.setKylinConfig(kylinconfig);
+ queryParams.setSelect(true);
+ queryParams.setForcedToPushDown(true);
+
+ Assert.assertThrows(NotSupportedSQLException.class, () -> queryRoutingEngine.queryWithSqlMassage(queryParams));
+ }
}
diff --git a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/CuboidAggregator.scala b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/CuboidAggregator.scala
index e1f538bad8..f81e6f7d37 100644
--- a/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/CuboidAggregator.scala
+++ b/src/spark-project/engine-spark/src/main/scala/org/apache/kylin/engine/spark/job/CuboidAggregator.scala
@@ -20,19 +20,19 @@ package org.apache.kylin.engine.spark.job
import org.apache.kylin.common.KapConfig
import org.apache.kylin.engine.spark.builder.DFBuilderHelper.ENCODE_SUFFIX
+import org.apache.kylin.measure.bitmap.BitmapMeasureType
+import org.apache.kylin.measure.hllc.HLLCMeasureType
import org.apache.kylin.metadata.cube.cuboid.NSpanningTree
import org.apache.kylin.metadata.cube.model.{NCubeJoinedFlatTableDesc, NDataSegment}
import org.apache.kylin.metadata.model.NDataModel.Measure
-import org.apache.kylin.measure.bitmap.BitmapMeasureType
-import org.apache.kylin.measure.hllc.HLLCMeasureType
import org.apache.kylin.metadata.model.TblColRef
-import org.apache.spark.sql.functions.{col, _}
+import org.apache.spark.sql.catalyst.expressions.Literal
+import org.apache.spark.sql.functions._
import org.apache.spark.sql.types._
import org.apache.spark.sql.udaf._
import org.apache.spark.sql.util.SparderTypeUtil
import org.apache.spark.sql.util.SparderTypeUtil.toSparkType
-import org.apache.spark.sql.{Column, DataFrame, Dataset, Row, SparkSession}
-import org.apache.spark.sql.catalyst.expressions.Literal
+import org.apache.spark.sql.{Column, DataFrame}
import java.util
import java.util.Locale
@@ -60,20 +60,20 @@ object CuboidAggregator {
dataset.schema.fieldNames.zipWithIndex.map(tp => (tp._2, tp._1)).toMap
aggregate(dataset, dimensions, measures, //
- (colRef: TblColRef) => columnIndex.apply(flatTableDesc.getColumnIndex(colRef)) )
+ (colRef: TblColRef) => columnIndex.apply(flatTableDesc.getColumnIndex(colRef)))
}
/**
- * Avoid compilation error when invoking aggregate in java
- * incompatible types: Function1 is not a functional interface
- *
- * @param dataset
- * @param dimensions
- * @param measures
- * @param tableDesc
- * @param isSparkSQL
- * @return
- */
+ * Avoid compilation error when invoking aggregate in java
+ * incompatible types: Function1 is not a functional interface
+ *
+ * @param dataset
+ * @param dimensions
+ * @param measures
+ * @param tableDesc
+ * @param isSparkSQL
+ * @return
+ */
def aggregateJava(dataset: DataFrame,
dimensions: util.Set[Integer],
measures: util.Map[Integer, Measure],
@@ -223,6 +223,15 @@ object CuboidAggregator {
}
case "CORR" =>
new Column(Literal(null, DoubleType)).as(measureEntry._1.toString)
+ case "SUM_LC" =>
+ val colDataType = function.getReturnDataType
+ val sparkDataType = toSparkType(colDataType)
+ if (reuseLayout) {
+ new Column(ReuseSumLC(columns.head.expr, sparkDataType).toAggregateExpression()).as(measureEntry._1.toString)
+ } else {
+ new Column(EncodeSumLC(columns.head.expr, columns.drop(1).head.expr, sparkDataType)
+ .toAggregateExpression()).as(measureEntry._1.toString)
+ }
}
}.toSeq
diff --git a/src/spark-project/sparder/src/main/scala/org/apache/kylin/query/runtime/plan/AggregatePlan.scala b/src/spark-project/sparder/src/main/scala/org/apache/kylin/query/runtime/plan/AggregatePlan.scala
index 8a2682d460..714b42d15d 100644
--- a/src/spark-project/sparder/src/main/scala/org/apache/kylin/query/runtime/plan/AggregatePlan.scala
+++ b/src/spark-project/sparder/src/main/scala/org/apache/kylin/query/runtime/plan/AggregatePlan.scala
@@ -17,11 +17,11 @@
*/
package org.apache.kylin.query.runtime.plan
-import org.apache.kylin.engine.spark.utils.LogEx
import org.apache.calcite.rel.core.AggregateCall
import org.apache.calcite.rex.RexLiteral
import org.apache.calcite.sql.SqlKind
import org.apache.kylin.common.KylinConfig
+import org.apache.kylin.engine.spark.utils.LogEx
import org.apache.kylin.metadata.model.FunctionDesc
import org.apache.kylin.query.relnode.{KapAggregateRel, KapProjectRel, KylinAggregateCall, OLAPAggregateRel}
import org.apache.kylin.query.util.RuntimeHelper
@@ -43,7 +43,8 @@ import scala.collection.JavaConverters._
// scalastyle:off
object AggregatePlan extends LogEx {
val binaryMeasureType =
- List("PERCENTILE", "PERCENTILE_APPROX", "INTERSECT_COUNT", "COUNT_DISTINCT", "BITMAP_UUID", FunctionDesc.FUNC_BITMAP_BUILD)
+ List("PERCENTILE", "PERCENTILE_APPROX", "INTERSECT_COUNT", "COUNT_DISTINCT", "BITMAP_UUID",
+ FunctionDesc.FUNC_BITMAP_BUILD, FunctionDesc.FUNC_SUM_LC)
def agg(inputs: java.util.List[DataFrame],
rel: KapAggregateRel): DataFrame = logTime("aggregate", debug = true) {
@@ -83,6 +84,10 @@ object AggregatePlan extends LogEx {
case FunctionDesc.FUNC_BITMAP_BUILD =>
val aggName = SchemaProcessor.replaceToAggravateSchemaName(index, "BITMAP_BUILD_DECODE", hash, argNames: _*)
KapFunctions.precise_bitmap_build_decode(columnName.head).alias(aggName)
+ case FunctionDesc.FUNC_SUM_LC =>
+ val aggName = SchemaProcessor.replaceToAggravateSchemaName(index, "SUM_LC_DECODE", hash, argNames: _*)
+ val sparkDataType = SparderTypeUtil.toSparkType(dataType)
+ KapFunctions.k_sum_lc_decode(columnName.head, sparkDataType.json).alias(aggName)
case _ =>
col(schemaNames.apply(call.getArgList.get(0)))
}
@@ -185,6 +190,8 @@ object AggregatePlan extends LogEx {
} else if (funcName.equalsIgnoreCase(FunctionDesc.FUNC_PERCENTILE)) {
require(columnName.size == 2, s"Input columns size ${columnName.size} don't equal to 2.")
KapFunctions.k_percentile(columnName.head, columnName(1), dataType.getPrecision).alias(aggName)
+ } else if (funcName.equalsIgnoreCase(FunctionDesc.FUNC_SUM_LC)) {
+ KapFunctions.k_sum_lc(columnName.head, SparderTypeUtil.toSparkType(dataType)).alias(aggName)
} else {
callUDF(registeredFuncName, columnName.toList: _*).alias(aggName)
}
diff --git a/src/spark-project/sparder/src/main/scala/org/apache/spark/sql/KapFunctions.scala b/src/spark-project/sparder/src/main/scala/org/apache/spark/sql/KapFunctions.scala
index eae6251f1a..7d6d9e4cb2 100644
--- a/src/spark-project/sparder/src/main/scala/org/apache/spark/sql/KapFunctions.scala
+++ b/src/spark-project/sparder/src/main/scala/org/apache/spark/sql/KapFunctions.scala
@@ -21,13 +21,11 @@ import org.apache.spark.sql.catalyst.FunctionIdentifier
import org.apache.spark.sql.catalyst.analysis.FunctionRegistry.FunctionBuilder
import org.apache.spark.sql.catalyst.expressions.ExpressionUtils.expression
import org.apache.spark.sql.catalyst.expressions.aggregate.AggregateFunction
-import org.apache.spark.sql.catalyst.expressions.{ApproxCountDistinctDecode, CeilDateTime, DictEncode, DictEncodeV3, Expression, ExpressionInfo, FloorDateTime, ImplicitCastInputTypes, In, KapAddMonths, KapSubtractMonths, Like, Literal, PercentileDecode, PreciseCountDistinctDecode, RLike, RoundBase, SplitPart, Sum0, TimestampAdd, TimestampDiff, Truncate}
-import org.apache.spark.sql.types.{ArrayType, BinaryType, DoubleType, LongType, StringType}
import org.apache.spark.sql.catalyst.expressions.codegen.Block.BlockHelper
import org.apache.spark.sql.catalyst.expressions.codegen.{CodeGenerator, CodegenContext, ExprCode}
-import org.apache.spark.sql.catalyst.expressions.{ApproxCountDistinctDecode, CeilDateTime, DictEncode, EmptyRow, Expression, ExpressionInfo, FloorDateTime, ImplicitCastInputTypes, In, KapAddMonths, KapSubtractMonths, Like, Literal, PercentileDecode, PreciseCountDistinctDecode, RLike, RoundBase, SplitPart, Sum0, TimestampAdd, TimestampDiff, Truncate}
-import org.apache.spark.sql.types.{ArrayType, BinaryType, ByteType, DataType, DecimalType, DoubleType, FloatType, IntegerType, LongType, ShortType, StringType}
-import org.apache.spark.sql.udaf.{ApproxCountDistinct, IntersectCount, Percentile, PreciseBitmapBuildBase64Decode, PreciseBitmapBuildBase64WithIndex, PreciseBitmapBuildPushDown, PreciseCardinality, PreciseCountDistinct, PreciseCountDistinctAndArray, PreciseCountDistinctAndValue, ReusePreciseCountDistinct}
+import org.apache.spark.sql.catalyst.expressions.{ApproxCountDistinctDecode, CeilDateTime, DictEncode, DictEncodeV3, EmptyRow, Expression, ExpressionInfo, FloorDateTime, ImplicitCastInputTypes, In, KapAddMonths, KapSubtractMonths, Like, Literal, PercentileDecode, PreciseCountDistinctDecode, RLike, RoundBase, SplitPart, Sum0, SumLCDecode, TimestampAdd, TimestampDiff, Truncate}
+import org.apache.spark.sql.types._
+import org.apache.spark.sql.udaf._
object KapFunctions {
@@ -62,6 +60,12 @@ object KapFunctions {
def in(value: Expression, list: Seq[Expression]): Column = Column(In(value, list))
+ def k_sum_lc(measureCol: Column, wrapDataType: DataType): Column =
+ Column(ReuseSumLC(measureCol.expr, wrapDataType, wrapDataType).toAggregateExpression())
+
+ def k_sum_lc_decode(measureCol: Column, wrapDataType: String): Column =
+ Column(SumLCDecode(measureCol.expr, Literal(wrapDataType)))
+
def k_percentile(head: Column, column: Column, precision: Int): Column =
Column(Percentile(head.expr, precision, Some(column.expr), DoubleType).toAggregateExpression())
diff --git a/src/spark-project/sparder/src/main/scala/org/apache/spark/sql/catalyst/expressions/ExpressionUtils.scala b/src/spark-project/sparder/src/main/scala/org/apache/spark/sql/catalyst/expressions/ExpressionUtils.scala
index ea49dac47e..321bece62c 100644
--- a/src/spark-project/sparder/src/main/scala/org/apache/spark/sql/catalyst/expressions/ExpressionUtils.scala
+++ b/src/spark-project/sparder/src/main/scala/org/apache/spark/sql/catalyst/expressions/ExpressionUtils.scala
@@ -22,8 +22,8 @@ import org.apache.kylin.measure.hllc.HLLCounter
import org.apache.kylin.measure.percentile.PercentileSerializer
import org.apache.spark.sql.AnalysisException
import org.apache.spark.sql.catalyst.analysis.FunctionRegistry.FunctionBuilder
-import org.apache.spark.sql.types.Decimal
-import org.apache.spark.sql.udaf.BitmapSerAndDeSerObj
+import org.apache.spark.sql.types.{DataType, Decimal}
+import org.apache.spark.sql.udaf.{BitmapSerAndDeSerObj, SumLCUtil}
import java.nio.ByteBuffer
import scala.reflect.ClassTag
@@ -117,15 +117,15 @@ object ExpressionUtils {
}
def approxCountDistinctDecodeHelper(bytes: Any, precision: Any): Long = {
- val storageFormat = bytes.asInstanceOf[Array[Byte]]
- val preciseValue = precision.asInstanceOf[Int]
- if (storageFormat.nonEmpty) {
- val counter = new HLLCounter(preciseValue)
- counter.readRegisters(ByteBuffer.wrap(storageFormat))
- counter.getCountEstimate
- } else {
- 0L
- }
+ val storageFormat = bytes.asInstanceOf[Array[Byte]]
+ val preciseValue = precision.asInstanceOf[Int]
+ if (storageFormat.nonEmpty) {
+ val counter = new HLLCounter(preciseValue)
+ counter.readRegisters(ByteBuffer.wrap(storageFormat))
+ counter.getCountEstimate
+ } else {
+ 0L
+ }
}
def percentileDecodeHelper(bytes: Any, quantile: Any, precision: Any): Double = {
@@ -134,5 +134,12 @@ object ExpressionUtils {
val counter = serializer.deserialize(ByteBuffer.wrap(arrayBytes))
counter.getResultEstimateWithQuantileRatio(quantile.asInstanceOf[Decimal].toDouble)
}
-}
+ def sumLCDecodeHelper(bytes: Any, wrapDataType: Any): Number = {
+ val arrayBytes = bytes.asInstanceOf[Array[Byte]]
+ val codec = SumLCUtil.getNumericNullSafeSerializerByDataType(DataType.fromJson(wrapDataType.toString))
+ val counter = SumLCUtil.decodeToSumLCCounter(arrayBytes, codec)
+ counter.getSumLC
+ }
+
+}
diff --git a/src/spark-project/sparder/src/main/scala/org/apache/spark/sql/catalyst/expressions/KapExpresssions.scala b/src/spark-project/sparder/src/main/scala/org/apache/spark/sql/catalyst/expressions/KapExpresssions.scala
index 24e25c940f..ea28802be5 100644
--- a/src/spark-project/sparder/src/main/scala/org/apache/spark/sql/catalyst/expressions/KapExpresssions.scala
+++ b/src/spark-project/sparder/src/main/scala/org/apache/spark/sql/catalyst/expressions/KapExpresssions.scala
@@ -125,7 +125,6 @@ case class KapSubtractMonths(a: Expression, b: Expression)
import org.apache.spark.sql.catalyst.analysis.TypeCheckResult
import org.apache.spark.sql.catalyst.util.TypeUtils
-import org.apache.spark.sql.types._
@ExpressionDescription(
usage = "_FUNC_(expr) - Returns the sum calculated from values of a group. " +
@@ -772,4 +771,65 @@ case class PercentileDecode(bytes: Expression, quantile: Expression, precision:
val newChildren = Seq(newFirst, newSecond, newThird)
super.legacyWithNewChildren(newChildren)
}
-}
\ No newline at end of file
+}
+
+case class SumLCDecode(bytes: Expression, wrapDataTypeExpr: Expression) extends BinaryExpression with ExpectsInputTypes {
+ override def left: Expression = bytes;
+
+ override def right: Expression = wrapDataTypeExpr
+
+ override def inputTypes: Seq[AbstractDataType] = Seq(BinaryType, StringType)
+
+ def wrapDataType = DataType.fromJson(wrapDataTypeExpr.toString)
+
+ override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
+ val leftGen = left.genCode(ctx)
+ val rightGen = right.genCode(ctx)
+ val expressionUtils = ExpressionUtils.getClass.getName.stripSuffix("$")
+ val decimalUtil = classOf[Decimal].getName
+ val evalValue = ctx.freshName("evalValue")
+ val javaType = CodeGenerator.javaType(dataType)
+ val boxedJavaType = CodeGenerator.boxedType(javaType)
+ val commonCodeBlock =
+ code"""
+ ${leftGen.code}
+ ${rightGen.code}
+ Number $evalValue = $expressionUtils.sumLCDecodeHelper(${leftGen.value}, ${rightGen.value});
+ boolean ${ev.isNull} = $evalValue == null;
+ $javaType ${ev.value} = ${CodeGenerator.defaultValue(dataType)};
+ """
+ val conditionCodeBlock = if (wrapDataType.isInstanceOf[DecimalType]) {
+ code"""
+ if(!${ev.isNull}) {
+ ${ev.value} = $decimalUtil.fromDecimal($evalValue);
+ }
+ """
+ } else {
+ code"""
+ if(!${ev.isNull}) {
+ ${ev.value} = ($boxedJavaType) $evalValue;
+ }
+ """
+ }
+ ev.copy(code = commonCodeBlock + conditionCodeBlock)
+ }
+
+ override protected def nullSafeEval(bytes: Any, wrapDataTypeExpr: Any): Any = {
+ val decodeVal = ExpressionUtils.sumLCDecodeHelper(bytes, wrapDataTypeExpr)
+ wrapDataType match {
+ case DecimalType() =>
+ Decimal.fromDecimal(decodeVal.asInstanceOf[java.math.BigDecimal])
+ case _ =>
+ decodeVal
+ }
+ }
+
+ override def dataType: DataType = wrapDataType
+
+ override def prettyName: String = "sum_lc_decode"
+
+ override protected def withNewChildrenInternal(newLeft: Expression, newRight: Expression): Expression = {
+ val newChildren = Seq(newLeft, newRight)
+ super.legacyWithNewChildren(newChildren)
+ }
+}
diff --git a/src/spark-project/sparder/src/main/scala/org/apache/spark/sql/udf/SparderAggFun.scala b/src/spark-project/sparder/src/main/scala/org/apache/spark/sql/udf/SparderAggFun.scala
index 422951291d..1397251368 100644
--- a/src/spark-project/sparder/src/main/scala/org/apache/spark/sql/udf/SparderAggFun.scala
+++ b/src/spark-project/sparder/src/main/scala/org/apache/spark/sql/udf/SparderAggFun.scala
@@ -18,9 +18,6 @@
package org.apache.spark.sql.udf
-import java.nio.ByteBuffer
-import java.util
-
import com.google.common.collect.Maps
import org.apache.kylin.measure.MeasureAggregator
import org.apache.kylin.measure.bitmap.BitmapCounter
@@ -35,6 +32,9 @@ import org.apache.spark.sql.expressions.{MutableAggregationBuffer, UserDefinedAg
import org.apache.spark.sql.types._
import org.apache.spark.sql.util.SparderTypeUtil
+import java.nio.ByteBuffer
+import java.util
+
class SparderAggFun(funcName: String, dataTp: KyDataType)
extends UserDefinedAggregateFunction
with Logging {
@@ -89,7 +89,7 @@ class SparderAggFun(funcName: String, dataTp: KyDataType)
byteBuffer = ByteBuffer.allocate(1024 * 1024)
}
- val initVal = if (isCount) {
+ val initVal = if (isCount) {
// return 0 instead of null in case of no input
measureAggregator.reset()
byteBuffer.clear()
@@ -106,6 +106,7 @@ class SparderAggFun(funcName: String, dataTp: KyDataType)
override def update(buffer: MutableAggregationBuffer, input: Row): Unit = {
merge(buffer, input)
}
+
override def merge(buffer: MutableAggregationBuffer, input: Row): Unit = {
if (!input.isNullAt(0)) {
measureAggregator.reset()
@@ -167,7 +168,7 @@ class SparderAggFun(funcName: String, dataTp: KyDataType)
case _ => null
}
- ret
+ ret
}
}
diff --git a/src/spark-project/spark-common/src/main/scala/org/apache/spark/sql/LayoutEntityConverter.scala b/src/spark-project/spark-common/src/main/scala/org/apache/spark/sql/LayoutEntityConverter.scala
index 528c8cb520..429cdfbbe9 100644
--- a/src/spark-project/spark-common/src/main/scala/org/apache/spark/sql/LayoutEntityConverter.scala
+++ b/src/spark-project/spark-common/src/main/scala/org/apache/spark/sql/LayoutEntityConverter.scala
@@ -49,11 +49,11 @@ object LayoutEntityConverter {
bucketSpec = Option(bucketSp))
}
- def toSchema() : StructType = {
+ def toSchema(): StructType = {
genCuboidSchemaFromNCuboidLayout(layoutEntity)
}
- def toExactlySchema() : StructType = {
+ def toExactlySchema(): StructType = {
genCuboidSchemaFromNCuboidLayout(layoutEntity, true)
}
}
@@ -122,6 +122,7 @@ object LayoutEntityConverter {
genSparkStructField(i._1.toString, i._2)
}.toSeq ++ measures)
}
+
def genBucketSpec(layoutEntity: LayoutEntity, partitionColumn: Set[String]): Option[BucketSpec] = {
if (layoutEntity.getShardByColumns.isEmpty) {
Option(BucketSpec(layoutEntity.getBucketNum,
@@ -161,6 +162,7 @@ object LayoutEntityConverter {
case "COLLECT_SET" =>
val parameter = function.getParameters.get(0)
ArrayType(SparderTypeUtil.toSparkType(parameter.getColRef.getType))
+ case "SUM_LC" => BinaryType
case _ => SparderTypeUtil.toSparkType(function.getReturnDataType)
}
}
diff --git a/src/spark-project/spark-common/src/main/scala/org/apache/spark/sql/udaf/NullSafeValueSerializer.scala b/src/spark-project/spark-common/src/main/scala/org/apache/spark/sql/udaf/NullSafeValueSerializer.scala
index 75a035a344..92fd5dd0d5 100644
--- a/src/spark-project/spark-common/src/main/scala/org/apache/spark/sql/udaf/NullSafeValueSerializer.scala
+++ b/src/spark-project/spark-common/src/main/scala/org/apache/spark/sql/udaf/NullSafeValueSerializer.scala
@@ -18,11 +18,11 @@
package org.apache.spark.sql.udaf
+import org.apache.spark.unsafe.types.UTF8String
+
import java.io.{DataInput, DataOutput}
import java.nio.charset.StandardCharsets
-import org.apache.spark.unsafe.types.UTF8String
-
@SerialVersionUID(1)
sealed trait NullSafeValueSerializer {
final def serialize(output: DataOutput, value: Any): Unit = {
@@ -150,6 +150,29 @@ class StringSerializer extends NullSafeValueSerializer {
class DecimalSerializer extends NullSafeValueSerializer {
override def serialize0(output: DataOutput, value: Any): Unit = {
val decimal = value.asInstanceOf[BigDecimal]
+ DecimalCodecUtil.encode(decimal, output)
+ }
+
+ override def deSerialize0(input: DataInput, length: Int): Any = {
+ DecimalCodecUtil.decode(input)
+ }
+}
+
+@SerialVersionUID(1)
+class JavaBigDecimalSerializer extends NullSafeValueSerializer {
+ override def serialize0(output: DataOutput, value: Any): Unit = {
+ val decimal = BigDecimal.apply(value.asInstanceOf[java.math.BigDecimal])
+ DecimalCodecUtil.encode(decimal, output);
+ }
+
+ override def deSerialize0(input: DataInput, length: Int): Any = {
+ val decimal = DecimalCodecUtil.decode(input)
+ decimal.asInstanceOf[BigDecimal].bigDecimal
+ }
+}
+
+object DecimalCodecUtil {
+ def encode(decimal: BigDecimal, output: DataOutput): Unit = {
val bytes = decimal.toString().getBytes(StandardCharsets.UTF_8)
output.writeInt(1 + bytes.length)
output.writeByte(decimal.scale)
@@ -157,7 +180,7 @@ class DecimalSerializer extends NullSafeValueSerializer {
output.write(bytes)
}
- override def deSerialize0(input: DataInput, length: Int): Any = {
+ def decode(input: DataInput): Any = {
val scale = input.readByte()
val length = input.readInt()
val bytes = new Array[Byte](length)
@@ -166,4 +189,3 @@ class DecimalSerializer extends NullSafeValueSerializer {
decimal.setScale(scale)
}
}
-
diff --git a/src/spark-project/spark-common/src/main/scala/org/apache/spark/sql/udaf/SumLC.scala b/src/spark-project/spark-common/src/main/scala/org/apache/spark/sql/udaf/SumLC.scala
new file mode 100644
index 0000000000..bb68841e01
--- /dev/null
+++ b/src/spark-project/spark-common/src/main/scala/org/apache/spark/sql/udaf/SumLC.scala
@@ -0,0 +1,191 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.udaf
+
+import com.esotericsoftware.kryo.KryoException
+import com.esotericsoftware.kryo.io.{Input, KryoDataInput, KryoDataOutput, Output}
+import org.apache.kylin.common.util.DateFormat
+import org.apache.kylin.measure.sumlc.SumLCCounter
+import org.apache.spark.internal.Logging
+import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.catalyst.expressions.Expression
+import org.apache.spark.sql.catalyst.expressions.aggregate.{ImperativeAggregate, TypedImperativeAggregate}
+import org.apache.spark.sql.types._
+
+/**
+ * Build sum_lc measure, has two implements，
+ * for non-reuse sum_lc, two input, one for value column, another for date column
+ * for reuse sum_lc，construct measure from parent layout
+ *
+ * @param wrapDataType return serialize data type
+ * @param outputDataType return calculate data type, see Percentile outputType
+ * @param mutableAggBufferOffset default value
+ * @param inputAggBufferOffset default value
+ */
+sealed abstract class BaseSumLC(wrapDataType: DataType,
+ outputDataType: DataType = BinaryType,
+ mutableAggBufferOffset: Int = 0,
+ inputAggBufferOffset: Int = 0)
+ extends TypedImperativeAggregate[SumLCCounter] with Serializable with Logging {
+ lazy val serializer: NullSafeValueSerializer = SumLCUtil.getNumericNullSafeSerializerByDataType(wrapDataType)
+
+ override def prettyName: String = "sum_lc"
+
+ override def eval(buffer: SumLCCounter): Any = {
+ outputDataType match {
+ case BinaryType =>
+ serialize(buffer)
+ case DecimalType() =>
+ if (buffer.getSumLC != null) {
+ Decimal.fromDecimal(buffer.getSumLC.asInstanceOf[java.math.BigDecimal])
+ } else {
+ Decimal.ZERO
+ }
+ case _ =>
+ buffer.getSumLC
+ }
+ }
+
+ override def createAggregationBuffer(): SumLCCounter = new SumLCCounter()
+
+ override def merge(buffer: SumLCCounter, input: SumLCCounter): SumLCCounter = {
+ SumLCCounter.merge(buffer, input)
+ }
+
+ override def serialize(buffer: SumLCCounter): Array[Byte] = {
+ val array: Array[Byte] = new Array[Byte](1024 * 1024)
+ val output: Output = new Output(array)
+ serialize(buffer, array, output)
+ }
+
+ private def serialize(buffer: SumLCCounter, array: Array[Byte], output: Output): Array[Byte] = {
+ try {
+ if (buffer == null) {
+ Array.empty[Byte]
+ } else {
+ output.clear()
+ val out = new KryoDataOutput(output)
+ serializer.serialize(out, buffer.getSumLC)
+ out.writeLong(buffer.getTimestamp)
+ val mark = output.position()
+ output.close()
+ array.slice(0, mark)
+ }
+ } catch {
+ case th: KryoException if th.getMessage.contains("Buffer overflow") =>
+ logWarning(s"Resize buffer size to ${array.length * 2}")
+ val updateArray = new Array[Byte](array.length * 2)
+ output.setBuffer(updateArray)
+ serialize(buffer, updateArray, output)
+ case th =>
+ throw th
+ }
+ }
+
+ override def deserialize(bytes: Array[Byte]): SumLCCounter = {
+ SumLCUtil.decodeToSumLCCounter(bytes, serializer)
+ }
+
+ override def nullable: Boolean = false
+
+ override def dataType: DataType = outputDataType
+}
+
+case class EncodeSumLC(
+ evalCol: Expression,
+ dateCol: Expression,
+ wrapDataType: DataType,
+ outputDataType: DataType = BinaryType,
+ mutableAggBufferOffset: Int = 0,
+ inputAggBufferOffset: Int = 0)
+ extends BaseSumLC(wrapDataType, outputDataType, mutableAggBufferOffset, inputAggBufferOffset) {
+
+ override def update(buffer: SumLCCounter, input: InternalRow): SumLCCounter = {
+ val columnEvalVal = evalCol.eval(input)
+ val columnVal = columnEvalVal match {
+ case decimal: Decimal =>
+ decimal.toJavaBigDecimal
+ case _ =>
+ columnEvalVal.asInstanceOf[Number]
+ }
+ val dateValStr = String.valueOf(dateCol.eval(input)).trim
+ val timestampVal = DateFormat.stringToMillis(dateValStr)
+ SumLCCounter.merge(buffer, columnVal, timestampVal)
+ }
+
+ override def withNewMutableAggBufferOffset(newMutableAggBufferOffset: Int): ImperativeAggregate =
+ copy(mutableAggBufferOffset = newMutableAggBufferOffset)
+
+ override def withNewInputAggBufferOffset(newInputAggBufferOffset: Int): ImperativeAggregate =
+ copy(inputAggBufferOffset = newInputAggBufferOffset)
+
+ override def children: Seq[Expression] = Seq(evalCol, dateCol)
+
+ override protected def withNewChildrenInternal(newChildren: IndexedSeq[Expression]): Expression =
+ super.legacyWithNewChildren(newChildren)
+}
+
+case class ReuseSumLC(measure: Expression,
+ wrapDataType: DataType,
+ outputDataType: DataType = BinaryType,
+ mutableAggBufferOffset: Int = 0,
+ inputAggBufferOffset: Int = 0)
+ extends BaseSumLC(wrapDataType, outputDataType, mutableAggBufferOffset, inputAggBufferOffset) {
+
+ override def update(buffer: SumLCCounter, input: InternalRow): SumLCCounter = {
+ val evalCounter = deserialize(measure.eval(input).asInstanceOf[Array[Byte]])
+ SumLCCounter.merge(buffer, evalCounter)
+ }
+
+ override def withNewMutableAggBufferOffset(newMutableAggBufferOffset: Int): ImperativeAggregate =
+ copy(mutableAggBufferOffset = newMutableAggBufferOffset)
+
+ override def withNewInputAggBufferOffset(newInputAggBufferOffset: Int): ImperativeAggregate =
+ copy(inputAggBufferOffset = newInputAggBufferOffset)
+
+ override def children: Seq[Expression] = Seq(measure)
+
+ override protected def withNewChildrenInternal(newChildren: IndexedSeq[Expression]): Expression =
+ super.legacyWithNewChildren(newChildren)
+
+}
+
+object SumLCUtil extends Logging {
+
+ def decodeToSumLCCounter(bytes: Array[Byte], codec: NullSafeValueSerializer): SumLCCounter = {
+ if (bytes.nonEmpty) {
+ val in = new KryoDataInput(new Input(bytes))
+ val sumLC = codec.deserialize(in).asInstanceOf[Number]
+ val timestamp = in.readLong()
+ new SumLCCounter(sumLC, timestamp)
+ } else {
+ new SumLCCounter()
+ }
+ }
+
+ def getNumericNullSafeSerializerByDataType(dataType: org.apache.spark.sql.types.DataType): NullSafeValueSerializer = {
+ dataType match {
+ case LongType => new LongSerializer
+ case DoubleType => new DoubleSerializer
+ case DecimalType() => new JavaBigDecimalSerializer
+ case dt => throw new UnsupportedOperationException("Unsupported sum_lc dimension type: " + dt)
+ }
+ }
+
+}
[kylin] 15/17: KYLIN-5398 Flame graph availability fix
Posted by xx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit d3ecc5a0847da26178e84cf13d55c0378764452e
Author: Guoliang Sun <gu...@kyligence.io>
AuthorDate: Thu Nov 10 15:13:06 2022 +0800
KYLIN-5398 Flame graph availability fix
---
.../libasyncProfiler-linux-arm64.so | Bin 0 -> 298256 bytes
.../libasyncProfiler-linux-x64.so | Bin 0 -> 303903 bytes
build/async-profiler-lib/libasyncProfiler-mac.so | Bin 0 -> 634272 bytes
build/release/compress.sh | 5 ++
.../java/org/apache/kylin/common/KapConfig.java | 13 +++-
.../org/apache/kylin/common/KylinConfigBase.java | 11 ++-
.../kylin/common/asyncprofiler/AsyncArchUtil.java | 77 +++++++++++++++++++++
.../kylin/common/asyncprofiler/AsyncProfiler.java | 69 +++++++++++++-----
.../common/asyncprofiler/AsyncProfilerUtils.java | 4 ++
.../libasyncProfiler-linux-arm64.so | Bin 0 -> 298256 bytes
.../libasyncProfiler-linux-x64.so | Bin 0 -> 303903 bytes
.../async-profiler-lib/libasyncProfiler-mac.so | Bin 0 -> 634272 bytes
.../async-profiler-lib/linux64/libasyncProfiler.so | Bin 314098 -> 0 bytes
.../async-profiler-lib/macOS/libasyncProfiler.so | Bin 239460 -> 0 bytes
.../AsyncProfilerExecutorPlugin.scala | 8 ++-
.../common/asyncprofiler/AsyncProfilerTool.scala | 8 ++-
.../apache/kylin/common/KylinConfigBaseTest.java | 1 +
.../common/asyncprofiler/AsyncArchUtilTest.java} | 34 ++++-----
.../common/asyncprofiler/AsyncProfilerTest.java | 17 +++--
.../asyncprofiler/AsyncProfilerToolTest.java | 24 +++++++
.../asyncprofiler/AsyncProfilerUtilsTest.java | 16 +++++
.../org/apache/kylin/rest/service/JobService.java | 3 +-
.../localmeta/lib/libasyncProfiler-linux-arm64.so | Bin 0 -> 298256 bytes
.../localmeta/lib/libasyncProfiler-linux-x64.so | Bin 0 -> 303903 bytes
.../localmeta/lib/libasyncProfiler-mac.so | Bin 0 -> 634272 bytes
.../kylin/engine/spark/job/NSparkExecutable.java | 55 +++++++++------
.../kylin/query/asyncprofiler/AsyncProfiling.scala | 9 ++-
.../QueryAsyncProfilerDriverPlugin.scala | 7 +-
.../query/asyncprofiler/AsyncPluginWithMeta.scala | 8 +--
.../query/asyncprofiler/AsyncProfilingTest.scala | 45 +++++-------
.../QueryAsyncProfilerDriverPluginTest.scala | 27 ++------
.../BuildAsyncProfilerDriverPlugin.scala | 9 ++-
32 files changed, 318 insertions(+), 132 deletions(-)
diff --git a/build/async-profiler-lib/libasyncProfiler-linux-arm64.so b/build/async-profiler-lib/libasyncProfiler-linux-arm64.so
new file mode 100644
index 0000000000..b959823506
Binary files /dev/null and b/build/async-profiler-lib/libasyncProfiler-linux-arm64.so differ
diff --git a/build/async-profiler-lib/libasyncProfiler-linux-x64.so b/build/async-profiler-lib/libasyncProfiler-linux-x64.so
new file mode 100644
index 0000000000..6d961cec0d
Binary files /dev/null and b/build/async-profiler-lib/libasyncProfiler-linux-x64.so differ
diff --git a/build/async-profiler-lib/libasyncProfiler-mac.so b/build/async-profiler-lib/libasyncProfiler-mac.so
new file mode 100644
index 0000000000..ad45237d5f
Binary files /dev/null and b/build/async-profiler-lib/libasyncProfiler-mac.so differ
diff --git a/build/release/compress.sh b/build/release/compress.sh
index 61c7f9eb9d..bf9d584b92 100755
--- a/build/release/compress.sh
+++ b/build/release/compress.sh
@@ -43,6 +43,11 @@ if [[ -d "influxdb" ]]; then
cp -rf postgresql ${package_name}/
fi
+# copy async profiler native files
+cp -rf async-profiler-lib/libasyncProfiler-mac.so "${package_name}"/lib/libasyncProfiler-mac.so
+cp -rf async-profiler-lib/libasyncProfiler-linux-x64.so "${package_name}"/lib/libasyncProfiler-linux-x64.so
+cp -rf async-profiler-lib/libasyncProfiler-linux-arm64.so "${package_name}"/lib/libasyncProfiler-linux-arm64.so
+
# Add ssb data preparation files
mkdir -p ${package_name}/tool/ssb
cp -rf ../src/examples/sample_cube/data ${package_name}/tool/ssb/
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/KapConfig.java b/src/core-common/src/main/java/org/apache/kylin/common/KapConfig.java
index 6cb7cbe9b2..06123ccf6c 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/KapConfig.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/KapConfig.java
@@ -22,12 +22,14 @@ import java.io.File;
import java.io.IOException;
import java.util.Map;
+import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.fs.Path;
import org.apache.kylin.common.annotation.Clarification;
import org.apache.kylin.common.util.EncryptUtil;
import org.apache.kylin.common.util.FileUtils;
+@Slf4j
@Clarification(priority = Clarification.Priority.MAJOR, msg = "Enterprise")
public class KapConfig {
@@ -248,7 +250,7 @@ public class KapConfig {
}
/**
- * Smart modeling
+ * Smart modeling
*/
public String getSmartModelingConf(String conf) {
return config.getOptional("kylin.smart.conf." + conf, null);
@@ -515,16 +517,21 @@ public class KapConfig {
public String sparderFiles() {
try {
- File storageFile = new File(getKylinConfig().getLogSparkExecutorPropertiesFile());
+ File storageFile = new File(config.getLogSparkExecutorPropertiesFile());
String additionalFiles = storageFile.getCanonicalPath();
- storageFile = new File(getKylinConfig().getLogSparkAppMasterPropertiesFile());
+ storageFile = new File(config.getLogSparkAppMasterPropertiesFile());
if (additionalFiles.isEmpty()) {
additionalFiles = storageFile.getCanonicalPath();
} else {
additionalFiles = additionalFiles + "," + storageFile.getCanonicalPath();
}
+ if (config.asyncProfilingEnabled()) {
+ additionalFiles = additionalFiles + "," + config.getAsyncProfilerFiles();
+ }
+ log.info("Sparder additionalFiles: {}", additionalFiles);
return config.getOptional("kylin.query.engine.sparder-additional-files", additionalFiles);
} catch (IOException e) {
+ log.error("Add sparderFiles failed, " + e);
return "";
}
}
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 0802fc242e..557901c837 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -19,6 +19,8 @@
package org.apache.kylin.common;
import static java.lang.Math.toIntExact;
+import static org.apache.kylin.common.asyncprofiler.AsyncProfiler.ASYNC_PROFILER_LIB_LINUX_ARM64;
+import static org.apache.kylin.common.asyncprofiler.AsyncProfiler.ASYNC_PROFILER_LIB_LINUX_X64;
import static org.apache.kylin.common.constant.Constants.KYLIN_SOURCE_JDBC_CONNECTION_URL_KEY;
import static org.apache.kylin.common.constant.Constants.KYLIN_SOURCE_JDBC_DRIVER_KEY;
import static org.apache.kylin.common.constant.Constants.KYLIN_SOURCE_JDBC_PASS_KEY;
@@ -2636,6 +2638,13 @@ public abstract class KylinConfigBase implements Serializable {
return getLogPropertyFile("spark-appmaster-log4j.xml");
}
+ public String getAsyncProfilerFiles() throws IOException {
+ String kylinHome = getKylinHomeWithoutWarn();
+ File libX64 = new File(kylinHome + "/lib/" + ASYNC_PROFILER_LIB_LINUX_X64);
+ File libArm64 = new File(kylinHome + "/lib/" + ASYNC_PROFILER_LIB_LINUX_ARM64);
+ return libX64.getCanonicalPath() + "," + libArm64.getCanonicalPath();
+ }
+
private String getLogPropertyFile(String filename) {
String parentFolder;
if (isDevEnv()) {
@@ -3597,7 +3606,7 @@ public abstract class KylinConfigBase implements Serializable {
public boolean buildJobProfilingEnabled() {
return !Boolean.parseBoolean(System.getProperty("spark.local", FALSE))
- && Boolean.parseBoolean(getOptional("kylin.engine.async-profiler-enabled", FALSE));
+ && Boolean.parseBoolean(getOptional("kylin.engine.async-profiler-enabled", TRUE));
}
public long buildJobProfilingResultTimeout() {
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/asyncprofiler/AsyncArchUtil.java b/src/core-common/src/main/java/org/apache/kylin/common/asyncprofiler/AsyncArchUtil.java
new file mode 100644
index 0000000000..a1b77ab622
--- /dev/null
+++ b/src/core-common/src/main/java/org/apache/kylin/common/asyncprofiler/AsyncArchUtil.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.common.asyncprofiler;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.stream.Stream;
+
+public class AsyncArchUtil {
+
+ public enum ArchType {
+ LINUX_X64(), LINUX_ARM64()
+ }
+
+ private static final Map<String, ArchType> ARCH_TO_PROCESSOR = new HashMap<>();
+
+ static {
+ addProcessors(ArchType.LINUX_X64, "x86_64", "amd64");
+ addProcessors(ArchType.LINUX_ARM64, "aarch64");
+ }
+
+ public static ArchType getProcessor() {
+ return getProcessor(getSystemProperty("os.arch"));
+ }
+
+ /**
+ * Returns a {@link ArchType} object with the given value {@link String}. The {@link String} must be
+ * like a value returned by the os.arch System Property.
+ *
+ * @param osArch A {@link String} like a value returned by the os.arch System Property.
+ * @return A {@link ArchType} when it exists, else {@code null}.
+ */
+ public static ArchType getProcessor(final String osArch) {
+ return ARCH_TO_PROCESSOR.get(osArch);
+ }
+
+ /**
+ * <p>
+ * Gets a System property, defaulting to {@code null} if the property cannot be read.
+ * </p>
+ * <p>
+ * If a {@code SecurityException} is caught, the return value is {@code null} and a message is written to
+ * {@code System.err}.
+ * </p>
+ *
+ * @param property the system property name
+ * @return the system property value or {@code null} if a security problem occurs
+ */
+ private static String getSystemProperty(final String property) {
+ try {
+ return System.getProperty(property);
+ } catch (final SecurityException ex) {
+ // we are not allowed to look at this property, the SystemUtils property value will default to null.
+ return null;
+ }
+ }
+
+ private static void addProcessors(ArchType archType, final String... keys) {
+ Stream.of(keys).forEach(key -> ARCH_TO_PROCESSOR.put(key, archType));
+ }
+}
\ No newline at end of file
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/asyncprofiler/AsyncProfiler.java b/src/core-common/src/main/java/org/apache/kylin/common/asyncprofiler/AsyncProfiler.java
index c49cdffe20..a0ba5f13a1 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/asyncprofiler/AsyncProfiler.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/asyncprofiler/AsyncProfiler.java
@@ -18,27 +18,31 @@
package org.apache.kylin.common.asyncprofiler;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
+import java.io.File;
import java.io.IOException;
+import java.nio.file.Files;
import java.util.Objects;
+import org.apache.kylin.common.KylinConfigBase;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
public class AsyncProfiler {
private static final Logger logger = LoggerFactory.getLogger(AsyncProfiler.class);
- private static final String LIB_FILE = "libasyncProfiler.so";
- private static final String LIB_PARENT = "/async-profiler-lib";
- private static final String MAC_LIB_PATH = LIB_PARENT + "/macOS/" + LIB_FILE;
- private static final String LINUX_64_LIB_PATH = LIB_PARENT + "/linux64/" + LIB_FILE;
+ // async profiler native files
+ public static final String ASYNC_PROFILER_LIB_MAC = "libasyncProfiler-mac.so";
+ public static final String ASYNC_PROFILER_LIB_LINUX_X64 = "libasyncProfiler-linux-x64.so";
+ public static final String ASYNC_PROFILER_LIB_LINUX_ARM64 = "libasyncProfiler-linux-arm64.so";
+ private static final String LIB_PARENT = "/async-profiler-lib/";
private static AsyncProfiler profiler;
private boolean loaded = false;
- public static synchronized AsyncProfiler getInstance() {
+ public static synchronized AsyncProfiler getInstance(boolean loadLocalLib) {
if (profiler == null) {
- profiler = new AsyncProfiler();
+ profiler = new AsyncProfiler(loadLocalLib);
}
return profiler;
}
@@ -55,14 +59,43 @@ public class AsyncProfiler {
logger.info("Test arg for ut: {}", ignore);
}
- private AsyncProfiler() {
+ private AsyncProfiler(boolean loadLocalLib) {
try {
boolean isTestingOnLocalMac = System.getProperty("os.name", "").contains("Mac")
|| System.getProperty("os.name", "").contains("OS X");
if (isTestingOnLocalMac) {
- loadLibAsyncProfilerSO(MAC_LIB_PATH);
+ loadLibAsyncProfilerSO(LIB_PARENT + ASYNC_PROFILER_LIB_MAC);
} else {
- loadLibAsyncProfilerSO(LINUX_64_LIB_PATH);
+ String libName;
+ File libPath;
+
+ // Select native lib loading based on machine architecture
+ AsyncArchUtil.ArchType archType = AsyncArchUtil.getProcessor();
+ logger.info("Machine's archType: {}", archType);
+ switch (archType) {
+ case LINUX_ARM64:
+ libName = ASYNC_PROFILER_LIB_LINUX_ARM64;
+ break;
+ case LINUX_X64:
+ default:
+ libName = ASYNC_PROFILER_LIB_LINUX_X64;
+ break;
+ }
+
+ // Adapting load paths based on Spark deployment patterns
+ if (loadLocalLib) {
+ libPath = new File(KylinConfigBase.getKylinHome() + "/lib/" + libName);
+ } else {
+ libPath = new File(libName);
+ }
+ logger.info("AsyncProfiler libPath: {}, exists: {}", libPath.getAbsolutePath(),
+ Files.exists(libPath.toPath()));
+ // check this for ut
+ if (libPath.exists()) {
+ System.load(libPath.getAbsolutePath());
+ } else {
+ loadLibAsyncProfilerSO(LIB_PARENT + libName);
+ }
}
loaded = true;
} catch (Exception e) {
@@ -71,12 +104,12 @@ public class AsyncProfiler {
}
private void loadLibAsyncProfilerSO(String libPath) throws IOException {
- final java.nio.file.Path tmpLib = java.io.File.createTempFile("libasyncProfiler", ".so").toPath();
- java.nio.file.Files.copy(
- Objects.requireNonNull(AsyncProfilerTool.class.getResourceAsStream(libPath)),
- tmpLib,
- java.nio.file.StandardCopyOption.REPLACE_EXISTING);
- System.load(tmpLib.toAbsolutePath().toString());
+ File asyncProfilerLib = File.createTempFile("libasyncProfiler", ".so");
+ java.nio.file.Files.copy(Objects.requireNonNull(AsyncProfilerTool.class.getResourceAsStream(libPath)),
+ asyncProfilerLib.toPath(), java.nio.file.StandardCopyOption.REPLACE_EXISTING);
+ logger.info("AsyncProfiler will try to load from libPath: {}, exists: {}", asyncProfilerLib.getAbsolutePath(),
+ asyncProfilerLib.exists());
+ System.load(asyncProfilerLib.getAbsolutePath());
}
public boolean isLoaded() {
diff --git a/src/core-common/src/main/java/org/apache/kylin/common/asyncprofiler/AsyncProfilerUtils.java b/src/core-common/src/main/java/org/apache/kylin/common/asyncprofiler/AsyncProfilerUtils.java
index e86648268a..cd74a7cd33 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/asyncprofiler/AsyncProfilerUtils.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/asyncprofiler/AsyncProfilerUtils.java
@@ -53,6 +53,10 @@ public class AsyncProfilerUtils {
this.cachedResult = countDownLatch;
}
+ public void build(File localCacheDir) {
+ this.localCacheDir = localCacheDir;
+ }
+
public void build(long resultCollectionTimeout, File localCacheDir) {
this.resultCollectionTimeout = resultCollectionTimeout;
this.localCacheDir = localCacheDir;
diff --git a/src/core-common/src/main/resources/async-profiler-lib/libasyncProfiler-linux-arm64.so b/src/core-common/src/main/resources/async-profiler-lib/libasyncProfiler-linux-arm64.so
new file mode 100644
index 0000000000..b959823506
Binary files /dev/null and b/src/core-common/src/main/resources/async-profiler-lib/libasyncProfiler-linux-arm64.so differ
diff --git a/src/core-common/src/main/resources/async-profiler-lib/libasyncProfiler-linux-x64.so b/src/core-common/src/main/resources/async-profiler-lib/libasyncProfiler-linux-x64.so
new file mode 100644
index 0000000000..6d961cec0d
Binary files /dev/null and b/src/core-common/src/main/resources/async-profiler-lib/libasyncProfiler-linux-x64.so differ
diff --git a/src/core-common/src/main/resources/async-profiler-lib/libasyncProfiler-mac.so b/src/core-common/src/main/resources/async-profiler-lib/libasyncProfiler-mac.so
new file mode 100644
index 0000000000..ad45237d5f
Binary files /dev/null and b/src/core-common/src/main/resources/async-profiler-lib/libasyncProfiler-mac.so differ
diff --git a/src/core-common/src/main/resources/async-profiler-lib/linux64/libasyncProfiler.so b/src/core-common/src/main/resources/async-profiler-lib/linux64/libasyncProfiler.so
deleted file mode 100755
index 4153f52868..0000000000
Binary files a/src/core-common/src/main/resources/async-profiler-lib/linux64/libasyncProfiler.so and /dev/null differ
diff --git a/src/core-common/src/main/resources/async-profiler-lib/macOS/libasyncProfiler.so b/src/core-common/src/main/resources/async-profiler-lib/macOS/libasyncProfiler.so
deleted file mode 100755
index c7298c6b82..0000000000
Binary files a/src/core-common/src/main/resources/async-profiler-lib/macOS/libasyncProfiler.so and /dev/null differ
diff --git a/src/core-common/src/main/scala/org/apache/kylin/common/asyncprofiler/AsyncProfilerExecutorPlugin.scala b/src/core-common/src/main/scala/org/apache/kylin/common/asyncprofiler/AsyncProfilerExecutorPlugin.scala
index fe4a3918a2..e423c98f81 100644
--- a/src/core-common/src/main/scala/org/apache/kylin/common/asyncprofiler/AsyncProfilerExecutorPlugin.scala
+++ b/src/core-common/src/main/scala/org/apache/kylin/common/asyncprofiler/AsyncProfilerExecutorPlugin.scala
@@ -20,6 +20,7 @@ package org.apache.kylin.common.asyncprofiler
import com.google.common.util.concurrent.ThreadFactoryBuilder
import org.apache.kylin.common.asyncprofiler.Message._
+import org.apache.kylin.common.util.ExecutorServiceUtil
import org.apache.spark.api.plugin.{ExecutorPlugin, PluginContext}
import org.apache.spark.internal.Logging
@@ -40,7 +41,8 @@ class AsyncProfilerExecutorPlugin extends ExecutorPlugin with Logging {
val profile = new Runnable {
override def run(): Unit = checkAndProfile()
}
- log.debug(s"AsyncProfiler status: ${AsyncProfilerTool.status()}")
+ AsyncProfilerTool.loadAsyncProfilerLib(false)
+ log.info(s"AsyncProfiler status: ${AsyncProfilerTool.status()}")
scheduledExecutorService.scheduleWithFixedDelay(
profile, 0, checkingInterval, TimeUnit.MILLISECONDS)
}
@@ -80,4 +82,8 @@ class AsyncProfilerExecutorPlugin extends ExecutorPlugin with Logging {
ctx.send(msg)
}
+ override def shutdown(): Unit = {
+ ExecutorServiceUtil.shutdownGracefully(scheduledExecutorService, 3)
+ super.shutdown()
+ }
}
diff --git a/src/core-common/src/main/scala/org/apache/kylin/common/asyncprofiler/AsyncProfilerTool.scala b/src/core-common/src/main/scala/org/apache/kylin/common/asyncprofiler/AsyncProfilerTool.scala
index 0deb703da9..69bb7c72bb 100644
--- a/src/core-common/src/main/scala/org/apache/kylin/common/asyncprofiler/AsyncProfilerTool.scala
+++ b/src/core-common/src/main/scala/org/apache/kylin/common/asyncprofiler/AsyncProfilerTool.scala
@@ -27,7 +27,13 @@ object AsyncProfilerTool {
val log: Logger = LoggerFactory.getLogger(AsyncProfilerTool.getClass)
- private val profiler = AsyncProfiler.getInstance()
+ private var profiler: AsyncProfiler = _
+
+ def loadAsyncProfilerLib(loadLocalLib: Boolean): Unit = {
+ // Local - load Sparder Driver or (Spark Driver which in client mode)
+ // Remote - load all Executors or (Spark Engine which in cluster mode)
+ profiler = AsyncProfiler.getInstance(loadLocalLib)
+ }
private var _running = false
diff --git a/src/core-common/src/test/java/org/apache/kylin/common/KylinConfigBaseTest.java b/src/core-common/src/test/java/org/apache/kylin/common/KylinConfigBaseTest.java
index f2c3cb1a2f..f60a2b003b 100644
--- a/src/core-common/src/test/java/org/apache/kylin/common/KylinConfigBaseTest.java
+++ b/src/core-common/src/test/java/org/apache/kylin/common/KylinConfigBaseTest.java
@@ -1191,6 +1191,7 @@ class KylinConfigBaseTest {
@Test
void testBuildJobProfilingEnabled() {
KylinConfig config = KylinConfig.getInstanceFromEnv();
+ config.setProperty("kylin.engine.async-profiler-enabled", "false");
assertFalse(config.buildJobProfilingEnabled());
config.setProperty("kylin.engine.async-profiler-enabled", "true");
assertTrue(config.buildJobProfilingEnabled());
diff --git a/src/spark-project/sparder/src/main/scala/org/apache/kylin/query/asyncprofiler/QueryAsyncProfilerDriverPlugin.scala b/src/core-common/src/test/java/org/apache/kylin/common/asyncprofiler/AsyncArchUtilTest.java
similarity index 50%
copy from src/spark-project/sparder/src/main/scala/org/apache/kylin/query/asyncprofiler/QueryAsyncProfilerDriverPlugin.scala
copy to src/core-common/src/test/java/org/apache/kylin/common/asyncprofiler/AsyncArchUtilTest.java
index 68f344186b..0ba16c4a75 100644
--- a/src/spark-project/sparder/src/main/scala/org/apache/kylin/query/asyncprofiler/QueryAsyncProfilerDriverPlugin.scala
+++ b/src/core-common/src/test/java/org/apache/kylin/common/asyncprofiler/AsyncArchUtilTest.java
@@ -16,29 +16,25 @@
* limitations under the License.
*/
-package org.apache.kylin.query.asyncprofiler
+package org.apache.kylin.common.asyncprofiler;
-import org.apache.spark.SparkContext
-import org.apache.spark.api.plugin.{DriverPlugin, PluginContext}
-import org.apache.spark.internal.Logging
+import org.junit.Assert;
+import org.junit.Test;
-import java.util
+public class AsyncArchUtilTest {
-class QueryAsyncProfilerDriverPlugin extends DriverPlugin with Logging {
-
- override def init(sc: SparkContext, pluginContext: PluginContext): util.Map[String, String] = super.init(sc, pluginContext)
+ @Test
+ public void testGetProcessor() {
+ AsyncArchUtil.ArchType archType = AsyncArchUtil.getProcessor();
+ Assert.assertNotNull(archType);
+ }
- override def receive(message: Any): AnyRef = {
- import org.apache.kylin.common.asyncprofiler.Message._
+ @Test
+ public void testArchType() {
+ AsyncArchUtil.ArchType archType = AsyncArchUtil.getProcessor("x86_64");
+ Assert.assertEquals(AsyncArchUtil.ArchType.LINUX_X64, archType);
- val (command, executorId, param) = processMessage(message.toString)
- command match {
- case NEXT_COMMAND =>
- AsyncProfiling.nextCommand()
- case RESULT =>
- AsyncProfiling.cacheExecutorResult(param, executorId)
- ""
- case _ => ""
+ archType = AsyncArchUtil.getProcessor("aarch64");
+ Assert.assertEquals(AsyncArchUtil.ArchType.LINUX_ARM64, archType);
}
- }
}
\ No newline at end of file
diff --git a/src/core-common/src/test/java/org/apache/kylin/common/asyncprofiler/AsyncProfilerTest.java b/src/core-common/src/test/java/org/apache/kylin/common/asyncprofiler/AsyncProfilerTest.java
index 82ab3f7a35..176d9ad119 100644
--- a/src/core-common/src/test/java/org/apache/kylin/common/asyncprofiler/AsyncProfilerTest.java
+++ b/src/core-common/src/test/java/org/apache/kylin/common/asyncprofiler/AsyncProfilerTest.java
@@ -26,8 +26,13 @@ import org.junit.Test;
public class AsyncProfilerTest {
@Test
- public void testLoaded() {
- Assert.assertTrue(AsyncProfiler.getInstance().isLoaded());
+ public void testLocalLoaded() {
+ Assert.assertTrue(AsyncProfiler.getInstance(true).isLoaded());
+ }
+
+ @Test
+ public void testRemoteLoaded() {
+ Assert.assertTrue(AsyncProfiler.getInstance(false).isLoaded());
}
// This may success in local Mac, but failed in CI
@@ -36,7 +41,7 @@ public class AsyncProfilerTest {
System.setProperty("os.name", "Mac");
String errorMsg = "";
try {
- AsyncProfiler.getInstance();
+ AsyncProfiler.getInstance(true);
} catch (Throwable throwable) {
errorMsg = throwable.getMessage();
}
@@ -45,7 +50,7 @@ public class AsyncProfilerTest {
@Test
public void testExecute() throws IOException {
- AsyncProfiler asyncProfiler = AsyncProfiler.getInstance();
+ AsyncProfiler asyncProfiler = AsyncProfiler.getInstance(true);
try {
asyncProfiler.execute("start,event=cpu");
asyncProfiler.stop();
@@ -59,12 +64,12 @@ public class AsyncProfilerTest {
@Test
public void testStop() {
Assert.assertThrows("Profiler is not active", IllegalStateException.class,
- AsyncProfiler.getInstance()::stop);
+ AsyncProfiler.getInstance(true)::stop);
}
@Test
public void testAsyncProfilerUtInstance() {
- AsyncProfiler originInstance = AsyncProfiler.getInstance();
+ AsyncProfiler originInstance = AsyncProfiler.getInstance(true);
AsyncProfiler utInstance = AsyncProfiler.utInstance();
Assert.assertSame(originInstance, utInstance);
}
diff --git a/src/core-common/src/test/java/org/apache/kylin/common/asyncprofiler/AsyncProfilerToolTest.java b/src/core-common/src/test/java/org/apache/kylin/common/asyncprofiler/AsyncProfilerToolTest.java
index 85b73a80bc..2cdb832339 100644
--- a/src/core-common/src/test/java/org/apache/kylin/common/asyncprofiler/AsyncProfilerToolTest.java
+++ b/src/core-common/src/test/java/org/apache/kylin/common/asyncprofiler/AsyncProfilerToolTest.java
@@ -40,6 +40,30 @@ public class AsyncProfilerToolTest extends NLocalFileMetadataTestCase {
this.cleanupTestMetadata();
}
+ @Test
+ public void testLoadLocalAsyncProfilerLib() {
+ AsyncProfilerTool.loadAsyncProfilerLib(true);
+ String errorMsg = "";
+ try {
+ AsyncProfilerTool.status();
+ } catch (Exception e) {
+ errorMsg = e.getMessage();
+ }
+ Assert.assertTrue(errorMsg.isEmpty());
+ }
+
+ @Test
+ public void testLoadRemoteAsyncProfilerLib() {
+ AsyncProfilerTool.loadAsyncProfilerLib(false);
+ String errorMsg = "";
+ try {
+ AsyncProfilerTool.status();
+ } catch (Exception e) {
+ errorMsg = e.getMessage();
+ }
+ Assert.assertTrue(errorMsg.isEmpty());
+ }
+
@Test
public void testStartAndStop() {
try {
diff --git a/src/core-common/src/test/java/org/apache/kylin/common/asyncprofiler/AsyncProfilerUtilsTest.java b/src/core-common/src/test/java/org/apache/kylin/common/asyncprofiler/AsyncProfilerUtilsTest.java
index 7eb636839a..fba1ad54a6 100644
--- a/src/core-common/src/test/java/org/apache/kylin/common/asyncprofiler/AsyncProfilerUtilsTest.java
+++ b/src/core-common/src/test/java/org/apache/kylin/common/asyncprofiler/AsyncProfilerUtilsTest.java
@@ -57,6 +57,22 @@ public class AsyncProfilerUtilsTest {
Assert.assertEquals(testFile, asyncProfilerUtilsBuild.localCacheDir);
}
+ @Test
+ public void testBuildWithNewLocalCacheDir() throws IOException {
+ AsyncProfilerUtils asyncProfilerUtilsBuild = AsyncProfilerUtils.getInstance();
+ asyncProfilerUtilsBuild.build(new CountDownLatch(2));
+ Assert.assertEquals(2, asyncProfilerUtilsBuild.cachedResult.getCount());
+
+ File testFile = Files.createTempDirectory("ke-build-async-test-profiler-").toFile();
+ asyncProfilerUtilsBuild.build(2L, testFile);
+ Assert.assertEquals(2L, asyncProfilerUtilsBuild.resultCollectionTimeout);
+ Assert.assertEquals(testFile, asyncProfilerUtilsBuild.localCacheDir);
+
+ testFile = Files.createTempDirectory("ke-build-async-test-profiler-").toFile();
+ asyncProfilerUtilsBuild.build(testFile);
+ Assert.assertEquals(testFile, asyncProfilerUtilsBuild.localCacheDir);
+ }
+
@Test
public void testWaitForResultTimeout() throws IOException, InterruptedException {
AsyncProfilerUtils asyncProfilerUtils = AsyncProfilerUtils.getInstance();
diff --git a/src/data-loading-service/src/main/java/org/apache/kylin/rest/service/JobService.java b/src/data-loading-service/src/main/java/org/apache/kylin/rest/service/JobService.java
index 22424681c5..494ac3b50e 100644
--- a/src/data-loading-service/src/main/java/org/apache/kylin/rest/service/JobService.java
+++ b/src/data-loading-service/src/main/java/org/apache/kylin/rest/service/JobService.java
@@ -47,7 +47,6 @@ import java.util.stream.Stream;
import javax.servlet.http.HttpServletRequest;
-import org.apache.kylin.metadata.epoch.EpochManager;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.MapUtils;
import org.apache.commons.io.IOUtils;
@@ -98,6 +97,7 @@ import org.apache.kylin.job.execution.StageBase;
import org.apache.kylin.metadata.cube.model.NBatchConstants;
import org.apache.kylin.metadata.cube.model.NDataSegment;
import org.apache.kylin.metadata.cube.model.NDataflowManager;
+import org.apache.kylin.metadata.epoch.EpochManager;
import org.apache.kylin.metadata.model.FusionModel;
import org.apache.kylin.metadata.model.FusionModelManager;
import org.apache.kylin.metadata.model.NDataModel;
@@ -1340,6 +1340,7 @@ public class JobService extends BasicService implements JobSupporter, ISmartAppl
}
public void setResponseLanguage(HttpServletRequest request) {
+ aclEvaluate.checkIsGlobalAdmin();
String languageToHandle = request.getHeader(HttpHeaders.ACCEPT_LANGUAGE);
if (languageToHandle == null) {
ErrorCode.setMsg("cn");
diff --git a/src/examples/test_case_data/localmeta/lib/libasyncProfiler-linux-arm64.so b/src/examples/test_case_data/localmeta/lib/libasyncProfiler-linux-arm64.so
new file mode 100644
index 0000000000..b959823506
Binary files /dev/null and b/src/examples/test_case_data/localmeta/lib/libasyncProfiler-linux-arm64.so differ
diff --git a/src/examples/test_case_data/localmeta/lib/libasyncProfiler-linux-x64.so b/src/examples/test_case_data/localmeta/lib/libasyncProfiler-linux-x64.so
new file mode 100644
index 0000000000..6d961cec0d
Binary files /dev/null and b/src/examples/test_case_data/localmeta/lib/libasyncProfiler-linux-x64.so differ
diff --git a/src/examples/test_case_data/localmeta/lib/libasyncProfiler-mac.so b/src/examples/test_case_data/localmeta/lib/libasyncProfiler-mac.so
new file mode 100644
index 0000000000..ad45237d5f
Binary files /dev/null and b/src/examples/test_case_data/localmeta/lib/libasyncProfiler-mac.so differ
diff --git a/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/job/NSparkExecutable.java b/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/job/NSparkExecutable.java
index a842c926ca..d31d02f83e 100644
--- a/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/job/NSparkExecutable.java
+++ b/src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/job/NSparkExecutable.java
@@ -18,13 +18,23 @@
package org.apache.kylin.engine.spark.job;
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-import com.google.common.collect.Sets;
-import org.apache.kylin.plugin.asyncprofiler.BuildAsyncProfilerSparkPlugin;
-import lombok.val;
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Paths;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Properties;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang.StringUtils;
@@ -60,26 +70,18 @@ import org.apache.kylin.metadata.cube.model.NDataflow;
import org.apache.kylin.metadata.cube.model.NDataflowManager;
import org.apache.kylin.metadata.project.EnhancedUnitOfWork;
import org.apache.kylin.metadata.project.NProjectManager;
+import org.apache.kylin.plugin.asyncprofiler.BuildAsyncProfilerSparkPlugin;
import org.apache.parquet.Strings;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.File;
-import java.io.IOException;
-import java.nio.file.Paths;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.LinkedHashSet;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Objects;
-import java.util.Optional;
-import java.util.Properties;
-import java.util.Set;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
+
+import lombok.val;
/**
*
@@ -883,6 +885,13 @@ public class NSparkExecutable extends AbstractExecutable implements ChainedStage
filePaths.add(kylinConf.getLogSparkAppMasterPropertiesFile());
filePaths.add(kylinConf.getLogSparkDriverPropertiesFile());
filePaths.add(kylinConf.getLogSparkExecutorPropertiesFile());
+ if (kylinConf.buildJobProfilingEnabled()) {
+ try {
+ filePaths.add(kylinConf.getAsyncProfilerFiles());
+ } catch (IOException e) {
+ logger.error("Add SparkPluginFile failed.", e);
+ }
+ }
filePaths.add(sparkConf.get(SPARK_FILES_1));
filePaths.add(sparkConf.get(SPARK_FILES_2));
diff --git a/src/spark-project/sparder/src/main/scala/org/apache/kylin/query/asyncprofiler/AsyncProfiling.scala b/src/spark-project/sparder/src/main/scala/org/apache/kylin/query/asyncprofiler/AsyncProfiling.scala
index e1dc51869c..d27cfc76f5 100644
--- a/src/spark-project/sparder/src/main/scala/org/apache/kylin/query/asyncprofiler/AsyncProfiling.scala
+++ b/src/spark-project/sparder/src/main/scala/org/apache/kylin/query/asyncprofiler/AsyncProfiling.scala
@@ -25,14 +25,14 @@ import org.apache.kylin.common.exception.{KylinException, QueryErrorCode}
import org.apache.spark.internal.Logging
import org.apache.spark.sql.SparderEnv
-import java.io.OutputStream
+import java.io.{File, OutputStream}
import java.nio.file.Files
import java.util.concurrent.CountDownLatch
object AsyncProfiling extends Logging {
- private val localCacheDir = Files.createTempDirectory("ke-async-profiler-result-").toFile
+ var localCacheDir: File = Files.createTempDirectory("ke-async-profiler-result-").toFile
localCacheDir.deleteOnExit()
private val resultCollectionTimeout = KylinConfig.getInstanceFromEnv.asyncProfilingResultTimeout
private val profilingTimeout = KylinConfig.getInstanceFromEnv.asyncProfilingProfileTimeout
@@ -57,6 +57,11 @@ object AsyncProfiling extends Logging {
throw new KylinException(QueryErrorCode.PROFILING_ALREADY_STARTED, "profiling is already started, stop it first")
}
logDebug("profiler start")
+ // Linux may periodically clean up the files in the /tmp directory
+ if (!localCacheDir.exists()) {
+ localCacheDir = Files.createTempDirectory("ke-async-profiler-result-").toFile
+ asyncProfilerUtils.build(localCacheDir)
+ }
asyncProfilerUtils.cleanLocalCache()
// expecting driver + count(executor) amount of results
cachedResult = new CountDownLatch(
diff --git a/src/spark-project/sparder/src/main/scala/org/apache/kylin/query/asyncprofiler/QueryAsyncProfilerDriverPlugin.scala b/src/spark-project/sparder/src/main/scala/org/apache/kylin/query/asyncprofiler/QueryAsyncProfilerDriverPlugin.scala
index 68f344186b..8af9631561 100644
--- a/src/spark-project/sparder/src/main/scala/org/apache/kylin/query/asyncprofiler/QueryAsyncProfilerDriverPlugin.scala
+++ b/src/spark-project/sparder/src/main/scala/org/apache/kylin/query/asyncprofiler/QueryAsyncProfilerDriverPlugin.scala
@@ -18,6 +18,7 @@
package org.apache.kylin.query.asyncprofiler
+import org.apache.kylin.common.asyncprofiler.AsyncProfilerTool
import org.apache.spark.SparkContext
import org.apache.spark.api.plugin.{DriverPlugin, PluginContext}
import org.apache.spark.internal.Logging
@@ -26,7 +27,11 @@ import java.util
class QueryAsyncProfilerDriverPlugin extends DriverPlugin with Logging {
- override def init(sc: SparkContext, pluginContext: PluginContext): util.Map[String, String] = super.init(sc, pluginContext)
+ override def init(sc: SparkContext, pluginContext: PluginContext): util.Map[String, String] = {
+ // Sparder Driver and KE are always in one JVM, in client mode
+ AsyncProfilerTool.loadAsyncProfilerLib(true)
+ super.init(sc, pluginContext)
+ }
override def receive(message: Any): AnyRef = {
import org.apache.kylin.common.asyncprofiler.Message._
diff --git a/src/spark-project/sparder/src/test/scala/org/apache/kylin/query/asyncprofiler/AsyncPluginWithMeta.scala b/src/spark-project/sparder/src/test/scala/org/apache/kylin/query/asyncprofiler/AsyncPluginWithMeta.scala
index d94a560dd8..9db5d72000 100644
--- a/src/spark-project/sparder/src/test/scala/org/apache/kylin/query/asyncprofiler/AsyncPluginWithMeta.scala
+++ b/src/spark-project/sparder/src/test/scala/org/apache/kylin/query/asyncprofiler/AsyncPluginWithMeta.scala
@@ -19,7 +19,6 @@
package org.apache.kylin.query.asyncprofiler
import org.apache.kylin.common.util.NLocalFileMetadataTestCase
-import org.apache.spark.sql.{SparderEnv, SparkSession}
import org.apache.spark.{SparkContext, SparkFunSuite}
import org.scalatest.BeforeAndAfterAll
@@ -44,11 +43,10 @@ trait AsyncPluginWithMeta extends SparkFunSuite with BeforeAndAfterAll {
}
protected def clearSparkSession(): Unit = {
- if (SparderEnv.isSparkAvailable) {
- SparderEnv.getSparkSession.stop()
+ if (sc != null) {
+ sc.stop()
+ sc = null
}
- SparkSession.setActiveSession(null)
- SparkSession.setDefaultSession(null)
}
override def beforeAll(): Unit = {
diff --git a/src/spark-project/sparder/src/test/scala/org/apache/kylin/query/asyncprofiler/AsyncProfilingTest.scala b/src/spark-project/sparder/src/test/scala/org/apache/kylin/query/asyncprofiler/AsyncProfilingTest.scala
index c05923664d..21c08c4ee7 100644
--- a/src/spark-project/sparder/src/test/scala/org/apache/kylin/query/asyncprofiler/AsyncProfilingTest.scala
+++ b/src/spark-project/sparder/src/test/scala/org/apache/kylin/query/asyncprofiler/AsyncProfilingTest.scala
@@ -24,7 +24,7 @@ import org.apache.spark.launcher.SparkLauncher
import org.apache.spark.{SparkConf, SparkContext}
import org.mockito.Mockito.mock
-import java.io.OutputStream
+import java.io.{File, OutputStream}
class AsyncProfilingTest extends AsyncPluginWithMeta {
@@ -34,55 +34,46 @@ class AsyncProfilingTest extends AsyncPluginWithMeta {
val statusFileName: String = flagFileDir + "/status"
val dumpFileName: String = flagFileDir + "/dump.tar.gz"
- test("init AsyncProfiling") {
- AsyncProfiling.asyncProfilerUtils
- }
-
- test("start and dump AsyncProfiling") {
+ override def beforeAll(): Unit = {
+ super.beforeAll()
val conf = new SparkConf()
.setAppName(getClass.getName)
.set(SparkLauncher.SPARK_MASTER, "local[1]")
.set("spark.plugins", sparkPluginName)
sc = new SparkContext(conf)
+ }
+
+ test("init AsyncProfiling") {
+ AsyncProfiling.asyncProfilerUtils
+ }
+
+ test("start and dump AsyncProfiling") {
AsyncProfiling.start("")
AsyncProfiling.dump("")
+ }
- sc.stop()
- sc = null
+ test("start with localCacheDir by delete") {
+ AsyncProfiling.nextCommand()
+ val localCacheDir = AsyncProfiling.localCacheDir
+ new File(localCacheDir.getAbsolutePath).delete()
+ AsyncProfiling.start("")
+ AsyncProfiling.dump("")
}
test("waitForResult AsyncProfiling") {
KylinConfig.getInstanceFromEnv.setProperty("kylin.query.async-profiler-result-timeout", "1ms")
- val conf = new SparkConf()
- .setAppName(getClass.getName)
- .set(SparkLauncher.SPARK_MASTER, "local[1]")
- .set("spark.plugins", sparkPluginName)
-
-
- sc = new SparkContext(conf)
AsyncProfiling.start("")
AsyncProfiling.dump("")
AsyncProfiling.waitForResult(mock(classOf[OutputStream]))
-
- sc.stop()
- sc = null
}
test("cacheExecutorResult AsyncProfiling") {
KylinConfig.getInstanceFromEnv.setProperty("kylin.query.async-profiler-result-timeout", "1ms")
- val conf = new SparkConf()
- .setAppName(getClass.getName)
- .set(SparkLauncher.SPARK_MASTER, "local[1]")
- .set("spark.plugins", sparkPluginName)
-
- sc = new SparkContext(conf)
AsyncProfiling.start("")
AsyncProfiling.cacheExecutorResult("content", "1")
-
- sc.stop()
- sc = null
+ AsyncProfiling.dump("")
}
}
diff --git a/src/spark-project/sparder/src/test/scala/org/apache/kylin/query/asyncprofiler/QueryAsyncProfilerDriverPluginTest.scala b/src/spark-project/sparder/src/test/scala/org/apache/kylin/query/asyncprofiler/QueryAsyncProfilerDriverPluginTest.scala
index 51843bfbbf..8d82c3c684 100644
--- a/src/spark-project/sparder/src/test/scala/org/apache/kylin/query/asyncprofiler/QueryAsyncProfilerDriverPluginTest.scala
+++ b/src/spark-project/sparder/src/test/scala/org/apache/kylin/query/asyncprofiler/QueryAsyncProfilerDriverPluginTest.scala
@@ -26,53 +26,36 @@ class QueryAsyncProfilerDriverPluginTest extends AsyncPluginWithMeta {
val sparkPluginName: String = classOf[QueryAsyncProfilerSparkPlugin].getName
- test("plugin initialization") {
+ override def beforeAll(): Unit = {
+ super.beforeAll()
val conf = new SparkConf()
.setAppName(getClass.getName)
.set(SparkLauncher.SPARK_MASTER, "local[1]")
.set("spark.plugins", sparkPluginName)
sc = new SparkContext(conf)
+ }
+
+ test("plugin initialization") {
Assert.assertEquals(sparkPluginName, sc.getConf.get("spark.plugins"))
new QueryAsyncProfilerDriverPlugin().receive("NEX-1:start,event=cpu")
-
- sc.stop()
- sc = null
}
test("plugin initialization receive result") {
- val conf = new SparkConf()
- .setAppName(getClass.getName)
- .set(SparkLauncher.SPARK_MASTER, "local[1]")
- .set("spark.plugins", sparkPluginName)
-
- sc = new SparkContext(conf)
Assert.assertEquals(sparkPluginName, sc.getConf.get("spark.plugins"))
try {
new QueryAsyncProfilerDriverPlugin().receive("RES-1:flamegraph")
} catch {
case _: Throwable =>
}
-
- sc.stop()
- sc = null
}
test("plugin initialization receive others") {
- val conf = new SparkConf()
- .setAppName(getClass.getName)
- .set(SparkLauncher.SPARK_MASTER, "local[1]")
- .set("spark.plugins", sparkPluginName)
-
- sc = new SparkContext(conf)
Assert.assertEquals(sparkPluginName, sc.getConf.get("spark.plugins"))
try {
new QueryAsyncProfilerDriverPlugin().receive("OTH-1:start,event=cpu")
} catch {
case _: Throwable =>
}
-
- sc.stop()
- sc = null
}
}
diff --git a/src/spark-project/spark-common/src/main/scala/org/apache/kylin/plugin/asyncprofiler/BuildAsyncProfilerDriverPlugin.scala b/src/spark-project/spark-common/src/main/scala/org/apache/kylin/plugin/asyncprofiler/BuildAsyncProfilerDriverPlugin.scala
index bbe900f49d..36497869e2 100644
--- a/src/spark-project/spark-common/src/main/scala/org/apache/kylin/plugin/asyncprofiler/BuildAsyncProfilerDriverPlugin.scala
+++ b/src/spark-project/spark-common/src/main/scala/org/apache/kylin/plugin/asyncprofiler/BuildAsyncProfilerDriverPlugin.scala
@@ -22,7 +22,7 @@ import com.google.common.util.concurrent.ThreadFactoryBuilder
import org.apache.hadoop.fs.{FSDataOutputStream, FileSystem, Path}
import org.apache.kylin.common.asyncprofiler.Message._
import org.apache.kylin.common.asyncprofiler.{AsyncProfilerTool, AsyncProfilerUtils}
-import org.apache.kylin.common.util.HadoopUtil
+import org.apache.kylin.common.util.{ExecutorServiceUtil, HadoopUtil}
import org.apache.spark.SparkContext
import org.apache.spark.api.plugin.{DriverPlugin, PluginContext}
import org.apache.spark.internal.Logging
@@ -71,7 +71,10 @@ class BuildAsyncProfilerDriverPlugin extends DriverPlugin with Logging {
val profile = new Runnable {
override def run(): Unit = checkAction()
}
- log.debug(s"AsyncProfiler status: ${AsyncProfilerTool.status()}")
+ val deployMode = sc.getConf.get("spark.submit.deployMode", "")
+ log.info("Current spark.submit.deployMode: {}", deployMode)
+ AsyncProfilerTool.loadAsyncProfilerLib(deployMode.equals("client"))
+ log.info(s"AsyncProfiler status: ${AsyncProfilerTool.status()}")
scheduledExecutorService.scheduleWithFixedDelay(
profile, 0, checkingInterval, TimeUnit.MILLISECONDS)
@@ -129,6 +132,8 @@ class BuildAsyncProfilerDriverPlugin extends DriverPlugin with Logging {
override def shutdown(): Unit = {
val fs: FileSystem = HadoopUtil.getFileSystem(statusFileName)
HadoopUtil.writeStringToHdfs(fs, ProfilerStatus.CLOSED, statusFileName)
+ ExecutorServiceUtil.shutdownGracefully(scheduledExecutorService, 3)
+ super.shutdown()
}
def start(params: String): Unit = {
[kylin] 17/17: [DIRTY] upgrade spark version
Posted by xx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 92af96e52afcf16fe712c13aea8715b05a6bc2b0
Author: ChenLiang <31...@users.noreply.github.com>
AuthorDate: Fri Nov 25 20:13:18 2022 +0800
[DIRTY] upgrade spark version
---
pom.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pom.xml b/pom.xml
index 40eb6b25f9..8f8fb7fd4f 100644
--- a/pom.xml
+++ b/pom.xml
@@ -64,7 +64,7 @@
<!-- Spark versions -->
<delta.version>1.2.1</delta.version>
- <spark.version>3.2.0-kylin-4.6.2.0-SNAPSHOT</spark.version>
+ <spark.version>3.2.0-kylin-4.6.2.0</spark.version>
<roaring.version>0.9.2-kylin-r4</roaring.version>
[kylin] 10/17: KYLIN-5353 add UT
Posted by xx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 092bbed4787b4db434b5cc06b356ec0a75462523
Author: Dorris Zhang <ru...@kyligence.io>
AuthorDate: Mon Oct 31 15:49:16 2022 +0800
KYLIN-5353 add UT
---
.../rest/service/ModelServiceBrokenRepairTest.java | 15 ++
.../3580693c-d1fd-7697-8ee8-0474f81c413d.json | 13 ++
.../722b027b-8906-379b-cf4f-ac2055ee528b.json | 2 +-
.../8dc4c289-c95d-0402-31e4-b123f9ef553f.json | 2 +-
...n => 3580693c-d1fd-7697-8ee8-0474f81c413d.json} | 8 +-
.../722b027b-8906-379b-cf4f-ac2055ee528b.json | 1 +
.../8dc4c289-c95d-0402-31e4-b123f9ef553f.json | 2 +-
.../3580693c-d1fd-7697-8ee8-0474f81c413d.json} | 172 ++++++++++++---------
.../722b027b-8906-379b-cf4f-ac2055ee528b.json | 2 +-
.../8dc4c289-c95d-0402-31e4-b123f9ef553f.json | 2 +-
.../metadata/ssb/table/SSB.CUSTOMER.json | 2 +-
.../metadata/ssb/table/SSB.LINEORDER.json | 2 +-
.../metadata/ssb/table/SSB.SUPPLIER.json | 2 +-
.../model_request_broken_aftr_reload.json | 2 +-
.../model_request_broken_aftr_reload_2.json | 2 +-
...2.json => model_request_with_timestamp_cc.json} | 172 ++++++++++++---------
16 files changed, 235 insertions(+), 166 deletions(-)
diff --git a/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelServiceBrokenRepairTest.java b/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelServiceBrokenRepairTest.java
index c91bf7a273..103ac90988 100644
--- a/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelServiceBrokenRepairTest.java
+++ b/src/modeling-service/src/test/java/org/apache/kylin/rest/service/ModelServiceBrokenRepairTest.java
@@ -29,6 +29,7 @@ import org.apache.kylin.metadata.favorite.FavoriteRuleManager;
import org.apache.kylin.metadata.model.MultiPartitionDesc;
import org.apache.kylin.metadata.model.NDataModel;
import org.apache.kylin.metadata.model.NDataModelManager;
+import org.apache.kylin.metadata.model.UpdateImpact;
import org.apache.kylin.metadata.recommendation.candidate.JdbcRawRecStore;
import org.apache.kylin.rest.config.initialize.ModelBrokenListener;
import org.apache.kylin.rest.request.ModelRequest;
@@ -168,4 +169,18 @@ public class ModelServiceBrokenRepairTest extends SourceTestCase {
Assert.assertEquals("LINEORDER.LO_ORDERDATE",
repairedDataModel.getMultiPartitionDesc().getColumns().getFirst());
}
+
+ @Test
+ public void testSaveModelWithTimestampCC() throws IOException {
+ String modelId = "3580693c-d1fd-7697-8ee8-0474f81c413d";
+ NDataModelManager modelManager = NDataModelManager.getInstance(getTestConfig(), PROJECT_NAME);
+ Assert.assertNotNull(modelManager.getDataModelDesc(modelId));
+ NDataModel dataModel = modelManager.getDataModelDesc(modelId);
+ ModelRequest modelRequest = JsonUtil.readValue(new File(
+ "src/test/resources/ut_meta/broken_repair_test/model_request/model_request_with_timestamp_cc.json"),
+ ModelRequest.class);
+ modelRequest.setProject(PROJECT_NAME);
+ UpdateImpact updateImpact = semanticService.updateModelColumns(dataModel, modelRequest);
+ Assert.assertEquals(0, updateImpact.getRemovedOrUpdatedCCs().size());
+ }
}
diff --git a/src/modeling-service/src/test/resources/ut_meta/broken_repair_test/metadata/ssb/dataflow/3580693c-d1fd-7697-8ee8-0474f81c413d.json b/src/modeling-service/src/test/resources/ut_meta/broken_repair_test/metadata/ssb/dataflow/3580693c-d1fd-7697-8ee8-0474f81c413d.json
new file mode 100644
index 0000000000..1eb6d06585
--- /dev/null
+++ b/src/modeling-service/src/test/resources/ut_meta/broken_repair_test/metadata/ssb/dataflow/3580693c-d1fd-7697-8ee8-0474f81c413d.json
@@ -0,0 +1,13 @@
+{
+ "uuid" : "3580693c-d1fd-7697-8ee8-0474f81c413d",
+ "last_modified" : 0,
+ "create_time" : 1667186632737,
+ "version" : "4.0.0.0",
+ "status" : "OFFLINE",
+ "last_status" : null,
+ "cost" : 50,
+ "query_hit_count" : 0,
+ "last_query_time" : 0,
+ "layout_query_hit_count" : { },
+ "segments" : [ ]
+}
diff --git a/src/modeling-service/src/test/resources/ut_meta/broken_repair_test/metadata/ssb/dataflow/722b027b-8906-379b-cf4f-ac2055ee528b.json b/src/modeling-service/src/test/resources/ut_meta/broken_repair_test/metadata/ssb/dataflow/722b027b-8906-379b-cf4f-ac2055ee528b.json
index cb1b91ff90..2ebf687089 100644
--- a/src/modeling-service/src/test/resources/ut_meta/broken_repair_test/metadata/ssb/dataflow/722b027b-8906-379b-cf4f-ac2055ee528b.json
+++ b/src/modeling-service/src/test/resources/ut_meta/broken_repair_test/metadata/ssb/dataflow/722b027b-8906-379b-cf4f-ac2055ee528b.json
@@ -10,4 +10,4 @@
"last_query_time" : 0,
"layout_query_hit_count" : { },
"segments" : [ ]
-}
\ No newline at end of file
+}
diff --git a/src/modeling-service/src/test/resources/ut_meta/broken_repair_test/metadata/ssb/dataflow/8dc4c289-c95d-0402-31e4-b123f9ef553f.json b/src/modeling-service/src/test/resources/ut_meta/broken_repair_test/metadata/ssb/dataflow/8dc4c289-c95d-0402-31e4-b123f9ef553f.json
index f5bd4046a6..f15f29f45a 100644
--- a/src/modeling-service/src/test/resources/ut_meta/broken_repair_test/metadata/ssb/dataflow/8dc4c289-c95d-0402-31e4-b123f9ef553f.json
+++ b/src/modeling-service/src/test/resources/ut_meta/broken_repair_test/metadata/ssb/dataflow/8dc4c289-c95d-0402-31e4-b123f9ef553f.json
@@ -10,4 +10,4 @@
"last_query_time" : 0,
"layout_query_hit_count" : { },
"segments" : [ ]
-}
\ No newline at end of file
+}
diff --git a/src/modeling-service/src/test/resources/ut_meta/broken_repair_test/metadata/ssb/index_plan/8dc4c289-c95d-0402-31e4-b123f9ef553f.json b/src/modeling-service/src/test/resources/ut_meta/broken_repair_test/metadata/ssb/index_plan/3580693c-d1fd-7697-8ee8-0474f81c413d.json
similarity index 79%
copy from src/modeling-service/src/test/resources/ut_meta/broken_repair_test/metadata/ssb/index_plan/8dc4c289-c95d-0402-31e4-b123f9ef553f.json
copy to src/modeling-service/src/test/resources/ut_meta/broken_repair_test/metadata/ssb/index_plan/3580693c-d1fd-7697-8ee8-0474f81c413d.json
index 587c63dc4c..71e295b00b 100644
--- a/src/modeling-service/src/test/resources/ut_meta/broken_repair_test/metadata/ssb/index_plan/8dc4c289-c95d-0402-31e4-b123f9ef553f.json
+++ b/src/modeling-service/src/test/resources/ut_meta/broken_repair_test/metadata/ssb/index_plan/3580693c-d1fd-7697-8ee8-0474f81c413d.json
@@ -1,7 +1,7 @@
{
- "uuid" : "8dc4c289-c95d-0402-31e4-b123f9ef553f",
- "last_modified" : 1658197554559,
- "create_time" : 1658197424087,
+ "uuid" : "3580693c-d1fd-7697-8ee8-0474f81c413d",
+ "last_modified" : 1667186632626,
+ "create_time" : 1667186632626,
"version" : "4.0.0.0",
"description" : null,
"rule_based_index" : null,
@@ -18,4 +18,4 @@
"layout_bucket_num" : { },
"approved_additional_recs" : 0,
"approved_removal_recs" : 0
-}
\ No newline at end of file
+}
diff --git a/src/modeling-service/src/test/resources/ut_meta/broken_repair_test/metadata/ssb/index_plan/722b027b-8906-379b-cf4f-ac2055ee528b.json b/src/modeling-service/src/test/resources/ut_meta/broken_repair_test/metadata/ssb/index_plan/722b027b-8906-379b-cf4f-ac2055ee528b.json
index 701d9cfa95..79896e6863 100644
--- a/src/modeling-service/src/test/resources/ut_meta/broken_repair_test/metadata/ssb/index_plan/722b027b-8906-379b-cf4f-ac2055ee528b.json
+++ b/src/modeling-service/src/test/resources/ut_meta/broken_repair_test/metadata/ssb/index_plan/722b027b-8906-379b-cf4f-ac2055ee528b.json
@@ -19,3 +19,4 @@
"approved_additional_recs" : 0,
"approved_removal_recs" : 0
}
+
diff --git a/src/modeling-service/src/test/resources/ut_meta/broken_repair_test/metadata/ssb/index_plan/8dc4c289-c95d-0402-31e4-b123f9ef553f.json b/src/modeling-service/src/test/resources/ut_meta/broken_repair_test/metadata/ssb/index_plan/8dc4c289-c95d-0402-31e4-b123f9ef553f.json
index 587c63dc4c..1b7e93786d 100644
--- a/src/modeling-service/src/test/resources/ut_meta/broken_repair_test/metadata/ssb/index_plan/8dc4c289-c95d-0402-31e4-b123f9ef553f.json
+++ b/src/modeling-service/src/test/resources/ut_meta/broken_repair_test/metadata/ssb/index_plan/8dc4c289-c95d-0402-31e4-b123f9ef553f.json
@@ -18,4 +18,4 @@
"layout_bucket_num" : { },
"approved_additional_recs" : 0,
"approved_removal_recs" : 0
-}
\ No newline at end of file
+}
diff --git a/src/modeling-service/src/test/resources/ut_meta/broken_repair_test/model_request/model_request_broken_aftr_reload_2.json b/src/modeling-service/src/test/resources/ut_meta/broken_repair_test/metadata/ssb/model_desc/3580693c-d1fd-7697-8ee8-0474f81c413d.json
similarity index 61%
copy from src/modeling-service/src/test/resources/ut_meta/broken_repair_test/model_request/model_request_broken_aftr_reload_2.json
copy to src/modeling-service/src/test/resources/ut_meta/broken_repair_test/metadata/ssb/model_desc/3580693c-d1fd-7697-8ee8-0474f81c413d.json
index bd499a2a65..ed870ed64e 100644
--- a/src/modeling-service/src/test/resources/ut_meta/broken_repair_test/model_request/model_request_broken_aftr_reload_2.json
+++ b/src/modeling-service/src/test/resources/ut_meta/broken_repair_test/metadata/ssb/model_desc/3580693c-d1fd-7697-8ee8-0474f81c413d.json
@@ -1,13 +1,13 @@
{
- "uuid" : "8dc4c289-c95d-0402-31e4-b123f9ef553f",
- "last_modified" : 1658197505312,
- "create_time" : 1658197424008,
+ "uuid" : "3580693c-d1fd-7697-8ee8-0474f81c413d",
+ "last_modified" : 1667186632623,
+ "create_time" : 1667186632149,
"version" : "4.0.0.0",
- "alias" : "model_broken_aftr_reload_2",
+ "alias" : "model1",
"owner" : "ADMIN",
"config_last_modifier" : null,
"config_last_modified" : 0,
- "description" : "",
+ "description" : null,
"fact_table" : "SSB.LINEORDER",
"fact_table_alias" : null,
"management_type" : "MODEL_BASED",
@@ -28,7 +28,7 @@
} ],
"filter_condition" : "",
"partition_desc" : {
- "partition_date_column" : "LINEORDER.LO_COMMITDATE",
+ "partition_date_column" : "LINEORDER.LO_ORDERDATE",
"partition_date_start" : 0,
"partition_date_format" : "yyyy-MM-dd",
"partition_type" : "APPEND",
@@ -43,113 +43,130 @@
"create_empty_segment_enabled" : false
},
"data_check_desc" : null,
- "semantic_version" : 1,
+ "semantic_version" : 0,
"storage_type" : 0,
"model_type" : "BATCH",
"all_named_columns" : [ {
"id" : 0,
- "name" : "LO_PARTKEY",
- "column" : "LINEORDER.LO_PARTKEY"
- }, {
- "id" : 1,
"name" : "LO_ORDERKEY",
"column" : "LINEORDER.LO_ORDERKEY",
"status" : "DIMENSION"
+ }, {
+ "id" : 1,
+ "name" : "LO_PARTKEY",
+ "column" : "LINEORDER.LO_PARTKEY",
+ "status" : "DIMENSION"
}, {
"id" : 2,
- "name" : "LO_ORDERDATE_TEST",
- "column" : "LINEORDER.LO_ORDERDATE_TEST",
- "status" : "TOMB"
+ "name" : "LO_DISCOUNT",
+ "column" : "LINEORDER.LO_DISCOUNT",
+ "status" : "DIMENSION"
}, {
"id" : 3,
- "name" : "LO_SHIPPRIOTITY",
- "column" : "LINEORDER.LO_SHIPPRIOTITY"
+ "name" : "LO_SUPPLYCOST",
+ "column" : "LINEORDER.LO_SUPPLYCOST",
+ "status" : "DIMENSION"
}, {
"id" : 4,
- "name" : "LO_ORDERPRIOTITY",
- "column" : "LINEORDER.LO_ORDERPRIOTITY"
+ "name" : "LO_COMMITDATE",
+ "column" : "LINEORDER.LO_COMMITDATE",
+ "status" : "DIMENSION"
}, {
"id" : 5,
- "name" : "LO_QUANTITY",
- "column" : "LINEORDER.LO_QUANTITY"
+ "name" : "LO_EXTENDEDPRICE",
+ "column" : "LINEORDER.LO_EXTENDEDPRICE",
+ "status" : "DIMENSION"
}, {
"id" : 6,
- "name" : "LO_COMMITDATE",
- "column" : "LINEORDER.LO_COMMITDATE"
+ "name" : "LO_TAX",
+ "column" : "LINEORDER.LO_TAX",
+ "status" : "DIMENSION"
}, {
"id" : 7,
- "name" : "LO_SHIPMODE",
- "column" : "LINEORDER.LO_SHIPMODE"
+ "name" : "LO_SUPPKEY",
+ "column" : "LINEORDER.LO_SUPPKEY",
+ "status" : "DIMENSION"
}, {
"id" : 8,
"name" : "LO_ORDTOTALPRICE",
- "column" : "LINEORDER.LO_ORDTOTALPRICE"
+ "column" : "LINEORDER.LO_ORDTOTALPRICE",
+ "status" : "DIMENSION"
}, {
"id" : 9,
- "name" : "LO_TAX",
- "column" : "LINEORDER.LO_TAX"
+ "name" : "LO_REVENUE",
+ "column" : "LINEORDER.LO_REVENUE",
+ "status" : "DIMENSION"
}, {
"id" : 10,
- "name" : "LO_LINENUMBER",
- "column" : "LINEORDER.LO_LINENUMBER",
+ "name" : "LO_ORDERDATE",
+ "column" : "LINEORDER.LO_ORDERDATE",
"status" : "DIMENSION"
}, {
"id" : 11,
- "name" : "LO_CUSTKEY_TEST",
- "column" : "LINEORDER.LO_CUSTKEY_TEST",
- "status" : "TOMB"
+ "name" : "LO_ORDERPRIOTITY",
+ "column" : "LINEORDER.LO_ORDERPRIOTITY",
+ "status" : "DIMENSION"
}, {
"id" : 12,
- "name" : "LO_EXTENDEDPRICE",
- "column" : "LINEORDER.LO_EXTENDEDPRICE"
+ "name" : "LO_SHIPPRIOTITY",
+ "column" : "LINEORDER.LO_SHIPPRIOTITY",
+ "status" : "DIMENSION"
}, {
"id" : 13,
- "name" : "LO_DISCOUNT",
- "column" : "LINEORDER.LO_DISCOUNT"
+ "name" : "LO_QUANTITY",
+ "column" : "LINEORDER.LO_QUANTITY",
+ "status" : "DIMENSION"
}, {
"id" : 14,
- "name" : "LO_REVENUE",
- "column" : "LINEORDER.LO_REVENUE"
+ "name" : "LO_SHIPMODE",
+ "column" : "LINEORDER.LO_SHIPMODE",
+ "status" : "DIMENSION"
}, {
"id" : 15,
- "name" : "LO_SUPPLYCOST",
- "column" : "LINEORDER.LO_SUPPLYCOST"
+ "name" : "LO_LINENUMBER",
+ "column" : "LINEORDER.LO_LINENUMBER",
+ "status" : "DIMENSION"
}, {
"id" : 16,
- "name" : "LO_SUPPKEY",
- "column" : "LINEORDER.LO_SUPPKEY"
+ "name" : "LO_CUSTKEY",
+ "column" : "LINEORDER.LO_CUSTKEY",
+ "status" : "DIMENSION"
}, {
"id" : 17,
- "name" : "C_PHONE",
- "column" : "CUSTOMER.C_PHONE"
+ "name" : "C_ADDRESS",
+ "column" : "CUSTOMER.C_ADDRESS"
}, {
"id" : 18,
- "name" : "C_CUSTKEY_TEST",
- "column" : "CUSTOMER.C_CUSTKEY_TEST"
- }, {
- "id" : 19,
"name" : "C_NATION",
"column" : "CUSTOMER.C_NATION"
}, {
- "id" : 20,
+ "id" : 19,
"name" : "C_CITY",
"column" : "CUSTOMER.C_CITY"
+ }, {
+ "id" : 20,
+ "name" : "C_PHONE",
+ "column" : "CUSTOMER.C_PHONE"
}, {
"id" : 21,
- "name" : "C_MKTSEGMENT",
- "column" : "CUSTOMER.C_MKTSEGMENT"
+ "name" : "C_REGION",
+ "column" : "CUSTOMER.C_REGION"
}, {
"id" : 22,
- "name" : "C_ADDRESS",
- "column" : "CUSTOMER.C_ADDRESS"
- }, {
- "id" : 23,
"name" : "C_NAME",
"column" : "CUSTOMER.C_NAME"
+ }, {
+ "id" : 23,
+ "name" : "C_MKTSEGMENT",
+ "column" : "CUSTOMER.C_MKTSEGMENT"
}, {
"id" : 24,
- "name" : "C_REGION",
- "column" : "CUSTOMER.C_REGION"
+ "name" : "C_CUSTKEY",
+ "column" : "CUSTOMER.C_CUSTKEY"
+ }, {
+ "id" : 25,
+ "name" : "CC111",
+ "column" : "LINEORDER.CC111"
} ],
"all_measures" : [ {
"name" : "COUNT_ALL",
@@ -168,31 +185,34 @@
"internal_ids" : [ ]
} ],
"recommendations_count" : 0,
- "computed_columns" : [ ],
+ "computed_columns" : [ {
+ "tableIdentity" : "SSB.LINEORDER",
+ "tableAlias" : "LINEORDER",
+ "columnName" : "CC111",
+ "expression" : "timestampadd(day, 1, LINEORDER.LO_ORDERDATE)",
+ "innerExpression" : "TIMESTAMPADD('day', 1, `LINEORDER`.`LO_ORDERDATE`)",
+ "datatype" : "DATE",
+ "comment" : null,
+ "rec_uuid" : null
+ } ],
"canvas" : {
"coordinate" : {
- "LINEORDER" : {
- "x" : 477.77777777777777,
- "y" : 7.5,
- "width" : 220.0,
- "height" : 200.0
- },
"CUSTOMER" : {
- "x" : 322.22222222222223,
- "y" : 233.05555555555557,
- "width" : 220.0,
- "height" : 200.0
+ "x" : 685.2777777777778,
+ "y" : 220.27777777777783,
+ "width" : 200.0,
+ "height" : 230.0
+ },
+ "LINEORDER" : {
+ "x" : 373.0555555555556,
+ "y" : 83.61111111111111,
+ "width" : 200.0,
+ "height" : 230.0
}
},
"zoom" : 9.0
},
- "broken_reason" : "SCHEMA",
- "handled_after_broken" : true,
- "multi_partition_desc" : {
- "columns" : [ "LINEORDER.LO_ORDERDATE" ],
- "partitions" : [ ],
- "max_partition_id" : -1
- },
+ "multi_partition_desc" : null,
"multi_partition_key_mapping" : null,
"fusion_id" : null
-}
\ No newline at end of file
+}
diff --git a/src/modeling-service/src/test/resources/ut_meta/broken_repair_test/metadata/ssb/model_desc/722b027b-8906-379b-cf4f-ac2055ee528b.json b/src/modeling-service/src/test/resources/ut_meta/broken_repair_test/metadata/ssb/model_desc/722b027b-8906-379b-cf4f-ac2055ee528b.json
index 5386211ef7..eb52a57689 100644
--- a/src/modeling-service/src/test/resources/ut_meta/broken_repair_test/metadata/ssb/model_desc/722b027b-8906-379b-cf4f-ac2055ee528b.json
+++ b/src/modeling-service/src/test/resources/ut_meta/broken_repair_test/metadata/ssb/model_desc/722b027b-8906-379b-cf4f-ac2055ee528b.json
@@ -457,4 +457,4 @@
"multi_partition_desc" : null,
"multi_partition_key_mapping" : null,
"fusion_id" : null
-}
\ No newline at end of file
+}
diff --git a/src/modeling-service/src/test/resources/ut_meta/broken_repair_test/metadata/ssb/model_desc/8dc4c289-c95d-0402-31e4-b123f9ef553f.json b/src/modeling-service/src/test/resources/ut_meta/broken_repair_test/metadata/ssb/model_desc/8dc4c289-c95d-0402-31e4-b123f9ef553f.json
index 9499eba6e4..b62ef430d3 100644
--- a/src/modeling-service/src/test/resources/ut_meta/broken_repair_test/metadata/ssb/model_desc/8dc4c289-c95d-0402-31e4-b123f9ef553f.json
+++ b/src/modeling-service/src/test/resources/ut_meta/broken_repair_test/metadata/ssb/model_desc/8dc4c289-c95d-0402-31e4-b123f9ef553f.json
@@ -195,4 +195,4 @@
},
"multi_partition_key_mapping" : null,
"fusion_id" : null
-}
\ No newline at end of file
+}
diff --git a/src/modeling-service/src/test/resources/ut_meta/broken_repair_test/metadata/ssb/table/SSB.CUSTOMER.json b/src/modeling-service/src/test/resources/ut_meta/broken_repair_test/metadata/ssb/table/SSB.CUSTOMER.json
index 793af290b2..823242f0bb 100644
--- a/src/modeling-service/src/test/resources/ut_meta/broken_repair_test/metadata/ssb/table/SSB.CUSTOMER.json
+++ b/src/modeling-service/src/test/resources/ut_meta/broken_repair_test/metadata/ssb/table/SSB.CUSTOMER.json
@@ -65,4 +65,4 @@
"transactional" : false,
"rangePartition" : false,
"partition_desc" : null
-}
\ No newline at end of file
+}
diff --git a/src/modeling-service/src/test/resources/ut_meta/broken_repair_test/metadata/ssb/table/SSB.LINEORDER.json b/src/modeling-service/src/test/resources/ut_meta/broken_repair_test/metadata/ssb/table/SSB.LINEORDER.json
index e6ec0011bc..291be55354 100644
--- a/src/modeling-service/src/test/resources/ut_meta/broken_repair_test/metadata/ssb/table/SSB.LINEORDER.json
+++ b/src/modeling-service/src/test/resources/ut_meta/broken_repair_test/metadata/ssb/table/SSB.LINEORDER.json
@@ -110,4 +110,4 @@
"transactional" : false,
"rangePartition" : false,
"partition_desc" : null
-}
\ No newline at end of file
+}
diff --git a/src/modeling-service/src/test/resources/ut_meta/broken_repair_test/metadata/ssb/table/SSB.SUPPLIER.json b/src/modeling-service/src/test/resources/ut_meta/broken_repair_test/metadata/ssb/table/SSB.SUPPLIER.json
index 4e73c1b25e..6b195ea133 100644
--- a/src/modeling-service/src/test/resources/ut_meta/broken_repair_test/metadata/ssb/table/SSB.SUPPLIER.json
+++ b/src/modeling-service/src/test/resources/ut_meta/broken_repair_test/metadata/ssb/table/SSB.SUPPLIER.json
@@ -60,4 +60,4 @@
"transactional" : false,
"rangePartition" : false,
"partition_desc" : null
-}
\ No newline at end of file
+}
diff --git a/src/modeling-service/src/test/resources/ut_meta/broken_repair_test/model_request/model_request_broken_aftr_reload.json b/src/modeling-service/src/test/resources/ut_meta/broken_repair_test/model_request/model_request_broken_aftr_reload.json
index 5143cf2297..c65de529e9 100644
--- a/src/modeling-service/src/test/resources/ut_meta/broken_repair_test/model_request/model_request_broken_aftr_reload.json
+++ b/src/modeling-service/src/test/resources/ut_meta/broken_repair_test/model_request/model_request_broken_aftr_reload.json
@@ -457,4 +457,4 @@
"multi_partition_desc" : null,
"multi_partition_key_mapping" : null,
"fusion_id" : null
-}
\ No newline at end of file
+}
diff --git a/src/modeling-service/src/test/resources/ut_meta/broken_repair_test/model_request/model_request_broken_aftr_reload_2.json b/src/modeling-service/src/test/resources/ut_meta/broken_repair_test/model_request/model_request_broken_aftr_reload_2.json
index bd499a2a65..b3e6f6832f 100644
--- a/src/modeling-service/src/test/resources/ut_meta/broken_repair_test/model_request/model_request_broken_aftr_reload_2.json
+++ b/src/modeling-service/src/test/resources/ut_meta/broken_repair_test/model_request/model_request_broken_aftr_reload_2.json
@@ -195,4 +195,4 @@
},
"multi_partition_key_mapping" : null,
"fusion_id" : null
-}
\ No newline at end of file
+}
diff --git a/src/modeling-service/src/test/resources/ut_meta/broken_repair_test/model_request/model_request_broken_aftr_reload_2.json b/src/modeling-service/src/test/resources/ut_meta/broken_repair_test/model_request/model_request_with_timestamp_cc.json
similarity index 61%
copy from src/modeling-service/src/test/resources/ut_meta/broken_repair_test/model_request/model_request_broken_aftr_reload_2.json
copy to src/modeling-service/src/test/resources/ut_meta/broken_repair_test/model_request/model_request_with_timestamp_cc.json
index bd499a2a65..ed870ed64e 100644
--- a/src/modeling-service/src/test/resources/ut_meta/broken_repair_test/model_request/model_request_broken_aftr_reload_2.json
+++ b/src/modeling-service/src/test/resources/ut_meta/broken_repair_test/model_request/model_request_with_timestamp_cc.json
@@ -1,13 +1,13 @@
{
- "uuid" : "8dc4c289-c95d-0402-31e4-b123f9ef553f",
- "last_modified" : 1658197505312,
- "create_time" : 1658197424008,
+ "uuid" : "3580693c-d1fd-7697-8ee8-0474f81c413d",
+ "last_modified" : 1667186632623,
+ "create_time" : 1667186632149,
"version" : "4.0.0.0",
- "alias" : "model_broken_aftr_reload_2",
+ "alias" : "model1",
"owner" : "ADMIN",
"config_last_modifier" : null,
"config_last_modified" : 0,
- "description" : "",
+ "description" : null,
"fact_table" : "SSB.LINEORDER",
"fact_table_alias" : null,
"management_type" : "MODEL_BASED",
@@ -28,7 +28,7 @@
} ],
"filter_condition" : "",
"partition_desc" : {
- "partition_date_column" : "LINEORDER.LO_COMMITDATE",
+ "partition_date_column" : "LINEORDER.LO_ORDERDATE",
"partition_date_start" : 0,
"partition_date_format" : "yyyy-MM-dd",
"partition_type" : "APPEND",
@@ -43,113 +43,130 @@
"create_empty_segment_enabled" : false
},
"data_check_desc" : null,
- "semantic_version" : 1,
+ "semantic_version" : 0,
"storage_type" : 0,
"model_type" : "BATCH",
"all_named_columns" : [ {
"id" : 0,
- "name" : "LO_PARTKEY",
- "column" : "LINEORDER.LO_PARTKEY"
- }, {
- "id" : 1,
"name" : "LO_ORDERKEY",
"column" : "LINEORDER.LO_ORDERKEY",
"status" : "DIMENSION"
+ }, {
+ "id" : 1,
+ "name" : "LO_PARTKEY",
+ "column" : "LINEORDER.LO_PARTKEY",
+ "status" : "DIMENSION"
}, {
"id" : 2,
- "name" : "LO_ORDERDATE_TEST",
- "column" : "LINEORDER.LO_ORDERDATE_TEST",
- "status" : "TOMB"
+ "name" : "LO_DISCOUNT",
+ "column" : "LINEORDER.LO_DISCOUNT",
+ "status" : "DIMENSION"
}, {
"id" : 3,
- "name" : "LO_SHIPPRIOTITY",
- "column" : "LINEORDER.LO_SHIPPRIOTITY"
+ "name" : "LO_SUPPLYCOST",
+ "column" : "LINEORDER.LO_SUPPLYCOST",
+ "status" : "DIMENSION"
}, {
"id" : 4,
- "name" : "LO_ORDERPRIOTITY",
- "column" : "LINEORDER.LO_ORDERPRIOTITY"
+ "name" : "LO_COMMITDATE",
+ "column" : "LINEORDER.LO_COMMITDATE",
+ "status" : "DIMENSION"
}, {
"id" : 5,
- "name" : "LO_QUANTITY",
- "column" : "LINEORDER.LO_QUANTITY"
+ "name" : "LO_EXTENDEDPRICE",
+ "column" : "LINEORDER.LO_EXTENDEDPRICE",
+ "status" : "DIMENSION"
}, {
"id" : 6,
- "name" : "LO_COMMITDATE",
- "column" : "LINEORDER.LO_COMMITDATE"
+ "name" : "LO_TAX",
+ "column" : "LINEORDER.LO_TAX",
+ "status" : "DIMENSION"
}, {
"id" : 7,
- "name" : "LO_SHIPMODE",
- "column" : "LINEORDER.LO_SHIPMODE"
+ "name" : "LO_SUPPKEY",
+ "column" : "LINEORDER.LO_SUPPKEY",
+ "status" : "DIMENSION"
}, {
"id" : 8,
"name" : "LO_ORDTOTALPRICE",
- "column" : "LINEORDER.LO_ORDTOTALPRICE"
+ "column" : "LINEORDER.LO_ORDTOTALPRICE",
+ "status" : "DIMENSION"
}, {
"id" : 9,
- "name" : "LO_TAX",
- "column" : "LINEORDER.LO_TAX"
+ "name" : "LO_REVENUE",
+ "column" : "LINEORDER.LO_REVENUE",
+ "status" : "DIMENSION"
}, {
"id" : 10,
- "name" : "LO_LINENUMBER",
- "column" : "LINEORDER.LO_LINENUMBER",
+ "name" : "LO_ORDERDATE",
+ "column" : "LINEORDER.LO_ORDERDATE",
"status" : "DIMENSION"
}, {
"id" : 11,
- "name" : "LO_CUSTKEY_TEST",
- "column" : "LINEORDER.LO_CUSTKEY_TEST",
- "status" : "TOMB"
+ "name" : "LO_ORDERPRIOTITY",
+ "column" : "LINEORDER.LO_ORDERPRIOTITY",
+ "status" : "DIMENSION"
}, {
"id" : 12,
- "name" : "LO_EXTENDEDPRICE",
- "column" : "LINEORDER.LO_EXTENDEDPRICE"
+ "name" : "LO_SHIPPRIOTITY",
+ "column" : "LINEORDER.LO_SHIPPRIOTITY",
+ "status" : "DIMENSION"
}, {
"id" : 13,
- "name" : "LO_DISCOUNT",
- "column" : "LINEORDER.LO_DISCOUNT"
+ "name" : "LO_QUANTITY",
+ "column" : "LINEORDER.LO_QUANTITY",
+ "status" : "DIMENSION"
}, {
"id" : 14,
- "name" : "LO_REVENUE",
- "column" : "LINEORDER.LO_REVENUE"
+ "name" : "LO_SHIPMODE",
+ "column" : "LINEORDER.LO_SHIPMODE",
+ "status" : "DIMENSION"
}, {
"id" : 15,
- "name" : "LO_SUPPLYCOST",
- "column" : "LINEORDER.LO_SUPPLYCOST"
+ "name" : "LO_LINENUMBER",
+ "column" : "LINEORDER.LO_LINENUMBER",
+ "status" : "DIMENSION"
}, {
"id" : 16,
- "name" : "LO_SUPPKEY",
- "column" : "LINEORDER.LO_SUPPKEY"
+ "name" : "LO_CUSTKEY",
+ "column" : "LINEORDER.LO_CUSTKEY",
+ "status" : "DIMENSION"
}, {
"id" : 17,
- "name" : "C_PHONE",
- "column" : "CUSTOMER.C_PHONE"
+ "name" : "C_ADDRESS",
+ "column" : "CUSTOMER.C_ADDRESS"
}, {
"id" : 18,
- "name" : "C_CUSTKEY_TEST",
- "column" : "CUSTOMER.C_CUSTKEY_TEST"
- }, {
- "id" : 19,
"name" : "C_NATION",
"column" : "CUSTOMER.C_NATION"
}, {
- "id" : 20,
+ "id" : 19,
"name" : "C_CITY",
"column" : "CUSTOMER.C_CITY"
+ }, {
+ "id" : 20,
+ "name" : "C_PHONE",
+ "column" : "CUSTOMER.C_PHONE"
}, {
"id" : 21,
- "name" : "C_MKTSEGMENT",
- "column" : "CUSTOMER.C_MKTSEGMENT"
+ "name" : "C_REGION",
+ "column" : "CUSTOMER.C_REGION"
}, {
"id" : 22,
- "name" : "C_ADDRESS",
- "column" : "CUSTOMER.C_ADDRESS"
- }, {
- "id" : 23,
"name" : "C_NAME",
"column" : "CUSTOMER.C_NAME"
+ }, {
+ "id" : 23,
+ "name" : "C_MKTSEGMENT",
+ "column" : "CUSTOMER.C_MKTSEGMENT"
}, {
"id" : 24,
- "name" : "C_REGION",
- "column" : "CUSTOMER.C_REGION"
+ "name" : "C_CUSTKEY",
+ "column" : "CUSTOMER.C_CUSTKEY"
+ }, {
+ "id" : 25,
+ "name" : "CC111",
+ "column" : "LINEORDER.CC111"
} ],
"all_measures" : [ {
"name" : "COUNT_ALL",
@@ -168,31 +185,34 @@
"internal_ids" : [ ]
} ],
"recommendations_count" : 0,
- "computed_columns" : [ ],
+ "computed_columns" : [ {
+ "tableIdentity" : "SSB.LINEORDER",
+ "tableAlias" : "LINEORDER",
+ "columnName" : "CC111",
+ "expression" : "timestampadd(day, 1, LINEORDER.LO_ORDERDATE)",
+ "innerExpression" : "TIMESTAMPADD('day', 1, `LINEORDER`.`LO_ORDERDATE`)",
+ "datatype" : "DATE",
+ "comment" : null,
+ "rec_uuid" : null
+ } ],
"canvas" : {
"coordinate" : {
- "LINEORDER" : {
- "x" : 477.77777777777777,
- "y" : 7.5,
- "width" : 220.0,
- "height" : 200.0
- },
"CUSTOMER" : {
- "x" : 322.22222222222223,
- "y" : 233.05555555555557,
- "width" : 220.0,
- "height" : 200.0
+ "x" : 685.2777777777778,
+ "y" : 220.27777777777783,
+ "width" : 200.0,
+ "height" : 230.0
+ },
+ "LINEORDER" : {
+ "x" : 373.0555555555556,
+ "y" : 83.61111111111111,
+ "width" : 200.0,
+ "height" : 230.0
}
},
"zoom" : 9.0
},
- "broken_reason" : "SCHEMA",
- "handled_after_broken" : true,
- "multi_partition_desc" : {
- "columns" : [ "LINEORDER.LO_ORDERDATE" ],
- "partitions" : [ ],
- "max_partition_id" : -1
- },
+ "multi_partition_desc" : null,
"multi_partition_key_mapping" : null,
"fusion_id" : null
-}
\ No newline at end of file
+}
[kylin] 11/17: KYLIN-5396 fix the version of pom in Kylin-it
Posted by xx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit a3a709853c2260ba4a8f113604f3b89bfc52a560
Author: Lex Meng <91...@users.noreply.github.com>
AuthorDate: Wed Nov 2 14:37:33 2022 +0800
KYLIN-5396 fix the version of pom in Kylin-it
* fix the version of pom in Kap-it/Kylin-it
* fix commons-beanutils version
---
pom.xml | 19 ++++++++++++-------
src/common-service/pom.xml | 1 -
src/kylin-it/pom.xml | 7 -------
3 files changed, 12 insertions(+), 15 deletions(-)
diff --git a/pom.xml b/pom.xml
index 6ad384d72d..80e6a6ed9c 100644
--- a/pom.xml
+++ b/pom.xml
@@ -758,6 +758,13 @@
<version>${project.version}</version>
<type>test-jar</type>
</dependency>
+ <dependency>
+ <groupId>org.apache.kylin</groupId>
+ <artifactId>kylin-streaming</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
<dependency>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin-common-service</artifactId>
@@ -1083,6 +1090,11 @@
<version>${guava.version}</version>
<scope>provided</scope>
</dependency>
+ <dependency>
+ <groupId>commons-beanutils</groupId>
+ <artifactId>commons-beanutils</artifactId>
+ <version>${beanutils.version}</version>
+ </dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>listenablefuture</artifactId>
@@ -2191,13 +2203,6 @@
<version>${servo-core.version}</version>
</dependency>
- <dependency>
- <groupId>org.springframework.boot</groupId>
- <artifactId>spring-boot-dependencies</artifactId>
- <version>${spring.boot.version}</version>
- <type>pom</type>
- <scope>import</scope>
- </dependency>
<dependency>
<groupId>de.codecentric</groupId>
<artifactId>spring-boot-admin-starter-client</artifactId>
diff --git a/src/common-service/pom.xml b/src/common-service/pom.xml
index 40c94752dc..6226d6ea78 100644
--- a/src/common-service/pom.xml
+++ b/src/common-service/pom.xml
@@ -166,7 +166,6 @@
<dependency>
<groupId>commons-beanutils</groupId>
<artifactId>commons-beanutils</artifactId>
- <version>${beanutils.version}</version>
</dependency>
<dependency>
diff --git a/src/kylin-it/pom.xml b/src/kylin-it/pom.xml
index 322af3ea24..19a5def3a6 100644
--- a/src/kylin-it/pom.xml
+++ b/src/kylin-it/pom.xml
@@ -12,11 +12,6 @@
<artifactId>kylin-it</artifactId>
- <properties>
- <beanutils.version>1.9.4</beanutils.version>
- <kylin.module.version>5.0.0-SNAPSHOT</kylin.module.version>
- </properties>
-
<dependencies>
<!--Springfox Swagger2 v2.9.2, swagger2markup v1.3.3. has specific requirement on guava and commons-beanutils -->
<dependency>
@@ -27,7 +22,6 @@
<dependency>
<groupId>commons-beanutils</groupId>
<artifactId>commons-beanutils</artifactId>
- <version>${beanutils.version}</version>
</dependency>
<dependency>
@@ -231,7 +225,6 @@
<dependency>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin-streaming</artifactId>
- <version>${kylin.module.version}</version>
<type>test-jar</type>
<scope>test</scope>
</dependency>