You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by xx...@apache.org on 2021/02/02 09:25:37 UTC
[kylin] branch kylin-on-parquet-v2 updated (8b75521 -> 39086e3)
This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a change to branch kylin-on-parquet-v2
in repository https://gitbox.apache.org/repos/asf/kylin.git.
discard 8b75521 [maven-release-plugin] prepare for next development iteration
discard 96796aa [maven-release-plugin] prepare release kylin-4.0.0-beta
discard 6a4dcc8 Fix maven-release-plugin.
omit aa67482 [maven-release-plugin] prepare for next development iteration
omit 1bd6714 [maven-release-plugin] prepare release kylin-4.0.0-beta
omit dc2fc39 Update version of shade-guava
omit 5650a4e HOTFIX Fix the issue of closing FileSystem
omit d7d2064 Fix storage clean up tool
omit 76492b5 Minor, fix replace-jars-under-spark.sh
omit 3aa72f3 KYLIN-4875 Remove executor configurations when execute resource detect step (local mode)
omit 588e08b KYLIN-4877 Use all dimension columns as sort columns when saving cuboid data
omit e719f7b KYLIn-4874 Fix CubeMigrationCLI bug for kylin4
omit dc5f295 Fix test case and fix system-cube.sh script
omit 5b3859c KYLIN-4653 Make the capacity for the LinkedBlockingQueue of BlockingReservoir configurable
omit 6f48ce8 KYLIN-4653 Make the capacity for the LinkedBlockingQueue of BlockingReservoir configurable
omit 3400338 KYLIN-4573 Add option to indicate whether to close file for every append for Hive Producer
omit 7431240 KYLIN-4496: Metric data missing
omit ad7b472 add test case
omit af3a1b5 KYLIN-4857 Refactor system cube for kylin4
omit 2ad58c7 KYLIN-4872 Fix NPE when there are more than one segment if cube planner is open
omit 124dd05 KYLIN-4818 Not store HLL binary file into RDBMS directly
omit b08c1be KYLIN-4859 Log4J reinitialized/reconfigured by Spark Logging (#1537)
omit 44402d3 KYLIN-4853 QueryPreparedStatementCache invalid in Spark Query Engine. (#1532)
omit 6f4e356 KYLIN-4858 Support Kylin4 deployment on CDH 6.X (#1535)
omit af8c8fe KYLIN-4818 Refine CuboidStatisticsJob to improve performance
omit 2e9597a KYLIN-4818 Reduce toString method call
omit d8e5db8 KYLIN-4818 Performance profile for CuboidStatisticsJob
omit 9e33d93 KYLIN-4818 Persist metadata in SparkExecutable
omit 3d6db49 KYLIN-4818 Support Cube Planner Phase One in Kylin 4
omit a6f78cd KYLIN-4818 Calculate cuboid rowcount via HLL
omit 32f8d66 fix KYLIN-4850, merge Cube's override kylin_properties into building job's configuration
This update removed existing revisions from the reference, leaving the
reference pointing at a previous point in the repository history.
* -- * -- N refs/heads/kylin-on-parquet-v2 (39086e3)
\
O -- O -- O (8b75521)
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
No new revisions were added by this update.
Summary of changes:
.github/workflows/maven.yml | 8 +-
.../kylin/engine/mr/common/CubeStatsReader.java | 59 +-
.../engine/mr/common/CuboidRecommenderUtil.java | 6 +-
.../engine/mr/common/StatisticsDecisionUtil.java | 8 +-
build/bin/find-hadoop-conf-dir.sh | 5 -
build/bin/find-hive-dependency.sh | 5 +-
build/bin/kylin.sh | 3 -
build/bin/replace-jars-under-spark.sh | 149 ----
build/bin/system-cube.sh | 38 +-
build/conf/kylin-server-log4j.properties | 11 +-
build/conf/spark-driver-log4j.properties | 2 +
.../org/apache/kylin/common/KylinConfigBase.java | 69 +-
.../org/apache/kylin/common/KylinConfigExt.java | 5 +-
.../kylin/common/annotation/Clarification.java | 6 +-
.../kylin/common/persistence/ResourceStore.java | 17 +-
.../org/apache/kylin/common/KylinConfigTest.java | 20 -
.../java/org/apache/kylin/cube/CubeSegment.java | 23 +-
.../cube/cuboid/algorithm/CuboidRecommender.java | 3 +-
.../cuboid/algorithm/greedy/GreedyAlgorithm.java | 16 +-
.../org/apache/kylin/cube/kv/CubeDimEncMap.java | 2 -
.../org/apache/kylin/measure/hllc/HLLCounter.java | 4 +-
.../apache/kylin/metrics/QuerySparkMetrics.java | 763 ---------------------
.../kylin/metrics/lib/impl/BlockingReservoir.java | 31 +-
.../kylin/metrics/lib/impl/MetricsSystem.java | 2 -
.../metrics/property/QuerySparkExecutionEnum.java | 80 ---
.../kylin/metrics/property/QuerySparkJobEnum.java | 66 --
.../metrics/property/QuerySparkStageEnum.java | 71 --
examples/test_case_data/localmeta/kylin.properties | 5 +-
external/pom.xml | 7 +-
.../common/logging/AbstractHdfsLogAppender.java | 33 +-
.../engine/spark/application/SparkApplication.java | 2 +-
.../engine/spark/builder/NBuildSourceInfo.java | 16 +-
.../engine/spark/job/NResourceDetectStep.java | 9 -
.../kylin/engine/spark/job/NSparkExecutable.java | 23 +-
.../engine/spark/utils/UpdateMetadataUtil.java | 38 +-
.../kylin/engine/spark/job/CubeBuildJob.java | 136 +---
.../kylin/engine/spark/job/CubeMergeJob.java | 4 +-
.../kylin/engine/spark/job/CuboidAggregator.scala | 1 -
.../engine/spark/job/CuboidStatisticsJob.scala | 167 -----
.../engine/spark/job/ParentSourceChooser.scala | 36 +-
.../kylin/engine/spark/utils/Repartitioner.java | 4 +-
.../kylin/engine/spark/metadata/MetaData.scala | 21 +-
.../engine/spark/metadata/MetadataConverter.scala | 14 +-
.../org/apache/spark/sql/SparderContext.scala | 6 -
.../spark/sql/metrics/SparderMetricsListener.scala | 144 ----
.../kylin/storage/hbase/HBaseResourceStore.java | 3 +-
.../kylin/metrics/lib/impl/hive/HiveProducer.java | 97 +--
.../kylin/rest/controller/CubeController.java | 56 +-
.../apache/kylin/rest/init/InitialTaskManager.java | 2 -
.../apache/kylin/rest/job/StorageCleanupJob.java | 27 +-
.../kylin/rest/metrics/QueryMetricsFacade.java | 166 ++++-
.../apache/kylin/rest/response/SQLResponse.java | 10 -
.../org/apache/kylin/rest/service/CubeService.java | 6 +-
.../kylin/rest/service/DashboardService.java | 20 +-
.../apache/kylin/rest/service/QueryService.java | 38 +-
.../kylin/rest/job/StorageCleanupJobTest.java | 27 +-
.../kylin/rest/response/SQLResponseTest.java | 9 +-
server/src/main/resources/kylinMetrics.xml | 9 +-
.../kylin/rest/metrics/QueryMetricsTest.java | 254 +------
.../org/apache/kylin/tool/CubeMigrationCLI.java | 13 +-
.../tool/metrics/systemcube/CubeDescCreator.java | 302 +++-----
.../metrics/systemcube/CubeInstanceCreator.java | 14 +-
.../tool/metrics/systemcube/HiveTableCreator.java | 136 ++--
.../tool/metrics/systemcube/KylinTableCreator.java | 20 +-
.../tool/metrics/systemcube/ModelCreator.java | 138 ++--
.../kylin/tool/metrics/systemcube/SCCreator.java | 18 +-
.../systemcube/streamingv2/KafkaTopicCreator.java | 6 +-
webapp/app/js/controllers/cube.js | 4 +-
webapp/app/js/services/cubes.js | 4 +-
webapp/app/partials/cubes/cube_detail.html | 2 +-
70 files changed, 657 insertions(+), 2862 deletions(-)
delete mode 100644 build/bin/replace-jars-under-spark.sh
delete mode 100644 core-metrics/src/main/java/org/apache/kylin/metrics/QuerySparkMetrics.java
delete mode 100644 core-metrics/src/main/java/org/apache/kylin/metrics/property/QuerySparkExecutionEnum.java
delete mode 100644 core-metrics/src/main/java/org/apache/kylin/metrics/property/QuerySparkJobEnum.java
delete mode 100644 core-metrics/src/main/java/org/apache/kylin/metrics/property/QuerySparkStageEnum.java
delete mode 100644 kylin-spark-project/kylin-spark-engine/src/main/scala/org/apache/kylin/engine/spark/job/CuboidStatisticsJob.scala
delete mode 100644 kylin-spark-project/kylin-spark-query/src/main/scala/org/apache/spark/sql/metrics/SparderMetricsListener.scala