You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2017/12/31 13:20:07 UTC
[50/50] [abbrv] kylin git commit: Merge commit
'5f2eff68d80ea6264d7590e14c052114c3cd6b74'
Merge commit '5f2eff68d80ea6264d7590e14c052114c3cd6b74'
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4d50b269
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4d50b269
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4d50b269
Branch: refs/heads/master
Commit: 4d50b26972bb7bbaff852172990e0f189f987673
Parents: 7889cec 5f2eff6
Author: Li Yang <li...@apache.org>
Authored: Sat Dec 30 21:57:51 2017 +0800
Committer: Li Yang <li...@apache.org>
Committed: Sun Dec 31 19:57:05 2017 +0800
----------------------------------------------------------------------
README.md | 2 +-
build/bin/load-hive-conf.sh | 4 +-
build/script/download-spark.sh | 18 +-
build/script/download-tomcat.sh | 4 +-
.../apache/kylin/common/KylinConfigBase.java | 149 ++++-
.../org/apache/kylin/common/QueryContext.java | 459 ++++++++++++-
.../org/apache/kylin/common/util/BytesUtil.java | 2 +-
.../apache/kylin/common/util/HadoopUtil.java | 19 +
.../org/apache/kylin/common/util/JsonUtil.java | 11 +-
.../java/org/apache/kylin/common/util/Pair.java | 16 +
.../kylin-backward-compatibility.properties | 1 +
.../main/resources/kylin-defaults.properties | 13 +
core-cube/pom.xml | 4 +
.../org/apache/kylin/cube/CubeDescManager.java | 4 +
.../org/apache/kylin/cube/CubeInstance.java | 90 ++-
.../java/org/apache/kylin/cube/CubeManager.java | 158 +++--
.../java/org/apache/kylin/cube/CubeUpdate.java | 14 +-
.../kylin/cube/common/RowKeySplitter.java | 2 +-
.../org/apache/kylin/cube/cuboid/Cuboid.java | 10 +-
.../apache/kylin/cube/cuboid/CuboidUtil.java | 85 +++
.../kylin/cube/cuboid/TreeCuboidScheduler.java | 3 +-
.../cuboid/algorithm/CuboidRecommender.java | 6 +
.../inmemcubing/AbstractInMemCubeBuilder.java | 13 +-
.../ConsumeBlockingQueueController.java | 84 +++
.../cube/inmemcubing/DoggedCubeBuilder.java | 128 +---
.../cube/inmemcubing/InMemCubeBuilder.java | 102 +--
.../InMemCubeBuilderInputConverter.java | 149 -----
.../kylin/cube/inmemcubing/InputConverter.java | 69 ++
.../cube/inmemcubing/InputConverterUnit.java | 42 ++
.../InputConverterUnitForBaseCuboid.java | 64 ++
.../InputConverterUnitForRawData.java | 169 +++++
.../RecordConsumeBlockingQueueController.java | 80 +++
.../org/apache/kylin/cube/kv/RowKeyDecoder.java | 4 +-
.../kylin/cube/model/AggregationGroup.java | 10 +-
.../kylin/cube/model/CubeBuildTypeEnum.java | 7 +-
.../org/apache/kylin/cube/model/CubeDesc.java | 111 +++-
.../validation/rule/AggregationGroupRule.java | 2 +-
.../kylin/gridtable/GTAggregateScanner.java | 8 +-
.../apache/kylin/gridtable/GTFilterScanner.java | 6 +
.../org/apache/kylin/gridtable/GTRecord.java | 5 +
.../kylin/cube/AggregationGroupRuleTest.java | 6 +-
.../org/apache/kylin/cube/CubeDescTest.java | 31 +-
.../org/apache/kylin/cube/CubeInstanceTest.java | 14 +-
.../org/apache/kylin/cube/CubeManagerTest.java | 109 +++
.../kylin/cube/cuboid/CuboidUtilTest.java | 57 ++
.../ConsumeBlockingQueueControllerTest.java | 124 ++++
.../apache/kylin/cube/kv/RowKeyDecoderTest.java | 2 +-
.../apache/kylin/cube/kv/RowKeyEncoderTest.java | 6 +-
.../apache/kylin/dict/DictionaryGenerator.java | 44 +-
.../org/apache/kylin/dict/DictionaryInfo.java | 4 +
.../apache/kylin/dict/DictionaryManager.java | 4 +
.../kylin/dict/GlobalDictionaryBuilder.java | 16 +-
.../apache/kylin/dict/IDictionaryBuilder.java | 2 +-
.../dict/MultipleDictionaryValueEnumerator.java | 10 +-
.../kylin/dict/Number2BytesConverter.java | 14 +-
.../global/SegmentAppendTrieDictBuilder.java | 9 +-
.../kylin/dict/DictionaryProviderTest.java | 2 +-
.../apache/kylin/dict/TrieDictionaryTest.java | 30 +-
core-job/pom.xml | 4 +
.../org/apache/kylin/engine/EngineFactory.java | 4 +
.../apache/kylin/engine/IBatchCubingEngine.java | 3 +
.../org/apache/kylin/job/JoinedFlatTable.java | 18 +-
.../kylin/job/common/ShellExecutable.java | 7 +-
.../kylin/job/constant/ExecutableConstants.java | 6 +
.../org/apache/kylin/job/dao/ExecutableDao.java | 18 +-
.../org/apache/kylin/job/dao/ExecutablePO.java | 11 +
.../kylin/job/engine/JobEngineConfig.java | 14 +
.../kylin/job/exception/ShellException.java | 43 ++
.../kylin/job/execution/AbstractExecutable.java | 50 +-
.../job/execution/CheckpointExecutable.java | 122 ++++
.../job/execution/DefaultChainedExecutable.java | 30 +-
.../kylin/job/execution/ExecutableManager.java | 46 +-
.../kylin/job/execution/ExecuteResult.java | 31 +
.../job/impl/threadpool/DefaultScheduler.java | 170 ++++-
.../kylin/job/metrics/JobMetricsFacade.java | 142 ++++
.../job/FiveSecondSucceedTestExecutable.java | 2 +-
.../kylin/job/RetryableTestExecutable.java | 50 ++
.../apache/kylin/job/RunningTestExecutable.java | 2 +-
.../apache/kylin/job/SelfStopExecutable.java | 2 +-
.../apache/kylin/job/SucceedTestExecutable.java | 2 +-
.../impl/threadpool/DefaultSchedulerTest.java | 16 +
.../kylin/measure/hllc/HLLCSerializer.java | 6 +-
.../kylin/metadata/TableMetadataManager.java | 2 +
.../metadata/datatype/BigDecimalSerializer.java | 3 +-
.../metadata/datatype/BooleanSerializer.java | 5 +-
.../metadata/datatype/DataTypeSerializer.java | 1 +
.../metadata/datatype/DateTimeSerializer.java | 5 +-
.../metadata/datatype/DoubleSerializer.java | 1 +
.../kylin/metadata/datatype/Int4Serializer.java | 5 +-
.../metadata/datatype/Long8Serializer.java | 5 +-
.../kylin/metadata/datatype/LongSerializer.java | 5 +-
.../metadata/datatype/StringSerializer.java | 1 +
.../kylin/metadata/model/DataModelDesc.java | 2 +-
.../kylin/metadata/model/FunctionDesc.java | 13 +-
.../kylin/metadata/model/ParameterDesc.java | 6 +-
.../kylin/metadata/model/PartitionDesc.java | 2 +-
.../apache/kylin/metadata/model/Segments.java | 26 +-
.../kylin/metadata/project/ProjectInstance.java | 94 ++-
.../hllc/NewHyperLogLogBenchmarkTest.java | 34 +-
core-metrics/pom.xml | 51 ++
.../apache/kylin/metrics/MetricsManager.java | 148 +++++
.../kylin/metrics/lib/ActiveReservoir.java | 40 ++
.../metrics/lib/ActiveReservoirFilter.java | 44 ++
.../metrics/lib/ActiveReservoirListener.java | 30 +
.../metrics/lib/ActiveReservoirReporter.java | 52 ++
.../org/apache/kylin/metrics/lib/Record.java | 51 ++
.../java/org/apache/kylin/metrics/lib/Sink.java | 23 +
.../org/apache/kylin/metrics/lib/SinkTool.java | 32 +
.../lib/impl/AbstractActiveReservoir.java | 68 ++
.../metrics/lib/impl/BaseScheduledReporter.java | 103 +++
.../metrics/lib/impl/BlockingReservoir.java | 167 +++++
.../metrics/lib/impl/InstantReservoir.java | 76 +++
.../kylin/metrics/lib/impl/MetricsSystem.java | 164 +++++
.../kylin/metrics/lib/impl/RecordEvent.java | 272 ++++++++
.../metrics/lib/impl/RecordEventTimeDetail.java | 77 +++
.../kylin/metrics/lib/impl/ReporterBuilder.java | 48 ++
.../kylin/metrics/lib/impl/StubReservoir.java | 54 ++
.../metrics/lib/impl/StubReservoirReporter.java | 51 ++
.../apache/kylin/metrics/lib/impl/StubSink.java | 30 +
.../metrics/lib/impl/TimePropertyEnum.java | 49 ++
.../metrics/lib/impl/TimedRecordEvent.java | 47 ++
.../kylin/metrics/property/JobPropertyEnum.java | 56 ++
.../metrics/property/QueryCubePropertyEnum.java | 54 ++
.../metrics/property/QueryPropertyEnum.java | 52 ++
.../metrics/property/QueryRPCPropertyEnum.java | 51 ++
.../apache/kylin/storage/StorageContext.java | 34 +-
.../storage/gtrecord/CubeScanRangePlanner.java | 49 +-
.../gtrecord/GTCubeStorageQueryBase.java | 35 +-
.../gtrecord/SequentialCubeTupleIterator.java | 6 +-
dev-support/make_patch.sh | 156 +++++
dev-support/python-requirements.txt | 21 +
dev-support/submit-patch.py | 311 +++++++++
engine-mr/pom.xml | 4 +
.../kylin/engine/mr/BatchCubingJobBuilder2.java | 29 +-
.../engine/mr/BatchOptimizeJobBuilder2.java | 228 +++++++
.../mr/BatchOptimizeJobCheckpointBuilder.java | 89 +++
.../org/apache/kylin/engine/mr/CubingJob.java | 104 ++-
.../org/apache/kylin/engine/mr/IMROutput2.java | 29 +-
.../kylin/engine/mr/JobBuilderSupport.java | 73 ++-
.../kylin/engine/mr/MRBatchCubingEngine2.java | 5 +
.../java/org/apache/kylin/engine/mr/MRUtil.java | 4 +
.../engine/mr/common/AbstractHadoopJob.java | 51 +-
.../kylin/engine/mr/common/BatchConstants.java | 8 +
.../kylin/engine/mr/common/CubeStatsReader.java | 147 +++--
.../kylin/engine/mr/common/CubeStatsWriter.java | 25 +-
.../engine/mr/common/CuboidRecommenderUtil.java | 22 +-
.../engine/mr/common/CuboidSchedulerUtil.java | 60 ++
.../engine/mr/common/CuboidStatsReaderUtil.java | 15 +-
.../kylin/engine/mr/common/HadoopCmdOutput.java | 9 +-
.../engine/mr/common/HadoopShellExecutable.java | 8 +-
.../engine/mr/common/JobInfoConverter.java | 67 +-
.../engine/mr/common/MapReduceExecutable.java | 9 +-
.../kylin/engine/mr/common/MapReduceUtil.java | 132 ++++
.../mr/common/SerializableConfiguration.java | 50 ++
.../mr/common/StatisticsDecisionUtil.java | 34 +-
.../mr/exception/HadoopShellException.java | 44 ++
.../engine/mr/exception/MapReduceException.java | 43 ++
.../mr/exception/SegmentNotFoundException.java | 44 ++
.../engine/mr/steps/BaseCuboidMapperBase.java | 2 +-
.../steps/CalculateStatsFromBaseCuboidJob.java | 120 ++++
.../CalculateStatsFromBaseCuboidMapper.java | 201 ++++++
...CalculateStatsFromBaseCuboidPartitioner.java | 59 ++
.../CalculateStatsFromBaseCuboidReducer.java | 115 ++++
.../engine/mr/steps/CopyDictionaryStep.java | 70 ++
.../engine/mr/steps/CreateDictionaryJob.java | 16 +-
.../engine/mr/steps/CubingExecutableUtil.java | 9 +
.../apache/kylin/engine/mr/steps/CuboidJob.java | 28 +-
.../mr/steps/FactDistinctColumnPartitioner.java | 51 +-
.../engine/mr/steps/FactDistinctColumnsJob.java | 37 +-
.../mr/steps/FactDistinctColumnsMapper.java | 332 ++++++----
.../mr/steps/FactDistinctColumnsMapperBase.java | 19 +-
.../mr/steps/FactDistinctColumnsReducer.java | 54 +-
.../FactDistinctColumnsReducerMapping.java | 156 +++++
.../mr/steps/FilterRecommendCuboidDataJob.java | 103 +++
.../steps/FilterRecommendCuboidDataMapper.java | 107 +++
.../mr/steps/InMemCuboidFromBaseCuboidJob.java | 160 +++++
.../steps/InMemCuboidFromBaseCuboidMapper.java | 102 +++
.../steps/InMemCuboidFromBaseCuboidReducer.java | 23 +
.../kylin/engine/mr/steps/InMemCuboidJob.java | 4 +-
.../engine/mr/steps/InMemCuboidMapper.java | 130 +---
.../engine/mr/steps/InMemCuboidMapperBase.java | 210 ++++++
.../kylin/engine/mr/steps/KVGTRecordWriter.java | 2 +-
.../engine/mr/steps/MergeCuboidMapper.java | 9 +-
.../engine/mr/steps/MergeDictionaryStep.java | 4 +-
.../engine/mr/steps/MergeStatisticsStep.java | 4 +-
.../mr/steps/MergeStatisticsWithOldStep.java | 155 +++++
.../kylin/engine/mr/steps/NDCuboidMapper.java | 15 +-
.../kylin/engine/mr/steps/ReducerNumSizing.java | 106 ---
.../engine/mr/steps/SaveStatisticsStep.java | 103 ++-
.../kylin/engine/mr/steps/UHCDictionaryJob.java | 154 +++++
.../engine/mr/steps/UHCDictionaryMapper.java | 101 +++
.../mr/steps/UHCDictionaryPartitioner.java | 30 +
.../engine/mr/steps/UHCDictionaryReducer.java | 113 ++++
.../mr/steps/UpdateCubeInfoAfterBuildStep.java | 4 +-
.../UpdateCubeInfoAfterCheckpointStep.java | 69 ++
.../mr/steps/UpdateCubeInfoAfterMergeStep.java | 8 +-
.../steps/UpdateCubeInfoAfterOptimizeStep.java | 72 ++
.../mr/steps/UpdateOldCuboidShardJob.java | 105 +++
.../mr/steps/UpdateOldCuboidShardMapper.java | 142 ++++
.../FactDistinctColumnsReducerMappingTest.java | 93 +++
.../engine/spark/SparkBatchCubingEngine.java | 67 --
.../kylin/engine/spark/SparkCountDemo.java | 80 ---
.../apache/kylin/engine/spark/SparkCubing.java | 590 -----------------
.../kylin/engine/spark/SparkCubingByLayer.java | 48 +-
.../engine/spark/SparkCubingJobBuilder.java | 69 --
.../kylin/engine/spark/SparkCuboidWriter.java | 29 -
.../kylin/engine/spark/SparkExecutable.java | 2 +-
.../kylin/engine/spark/SparkHelloWorld.java | 37 --
.../kylin/engine/spark/SparkHiveDemo.java | 52 --
.../engine/spark/cube/BufferedCuboidWriter.java | 106 ---
.../spark/cube/DefaultTupleConverter.java | 98 ---
.../spark/cube/ListBackedCuboidWriter.java | 59 --
.../kylin/engine/spark/cube/TupleConverter.java | 29 -
.../spark/cube/BufferedCuboidWriterTest.java | 69 --
.../test_case_data/localmeta/kylin.properties | 1 +
.../test_case_data/sandbox/kylin.properties | 15 +
.../ITDoggedCubeBuilderStressTest.java | 3 +-
.../inmemcubing/ITDoggedCubeBuilderTest.java | 8 +-
.../inmemcubing/ITInMemCubeBuilderTest.java | 27 +-
.../dict/ITGlobalDictionaryBuilderTest.java | 4 +-
.../apache/kylin/job/ContextTestExecutable.java | 2 +-
.../hbase/ITAclTableMigrationToolTest.java | 2 +
.../query/sql_intersect_count/query01.sql | 27 +
.../query/sql_intersect_count/query03.sql | 23 +
metrics-reporter-hive/pom.xml | 53 ++
.../metrics/lib/impl/hive/HiveProducer.java | 201 ++++++
.../lib/impl/hive/HiveProducerRecord.java | 196 ++++++
.../lib/impl/hive/HiveReservoirReporter.java | 139 ++++
.../kylin/metrics/lib/impl/hive/HiveSink.java | 30 +
metrics-reporter-kafka/pom.xml | 46 ++
.../kafka/KafkaActiveReserviorListener.java | 115 ++++
.../lib/impl/kafka/KafkaReservoirReporter.java | 139 ++++
.../kylin/metrics/lib/impl/kafka/KafkaSink.java | 29 +
pom.xml | 35 +-
.../org/apache/kylin/query/QueryConnection.java | 2 +-
.../query/adhoc/PushDownRunnerJdbcImpl.java | 45 +-
.../kylin/query/enumerator/OLAPQuery.java | 3 +
.../kylin/query/relnode/OLAPAggregateRel.java | 1 +
.../apache/kylin/query/relnode/OLAPContext.java | 2 +-
.../kylin/query/relnode/OLAPTableScan.java | 2 +-
.../kylin/query/security/QuerACLTestUtil.java | 43 --
.../kylin/query/security/QueryACLTestUtil.java | 55 ++
.../query/util/KeywordDefaultDirtyHack.java | 3 +-
.../query/util/KeywordDefaultDirtyHackTest.java | 60 ++
server-base/pom.xml | 12 +
.../kylin/rest/controller/CubeController.java | 277 +++++++-
.../rest/controller/DashboardController.java | 129 ++++
.../kylin/rest/controller/JobController.java | 5 +-
.../rest/controller/ProjectController.java | 16 +-
.../rest/controller/StreamingController.java | 46 +-
.../rest/job/StorageCleanJobHbaseUtil.java | 4 +-
.../kylin/rest/job/StorageCleanupJob.java | 18 +-
.../kylin/rest/metrics/QueryMetricsFacade.java | 165 ++++-
.../java/org/apache/kylin/rest/msg/Message.java | 12 +-
.../kylin/rest/request/JobOptimizeRequest.java | 34 +
.../apache/kylin/rest/request/SQLRequest.java | 2 +-
.../rest/response/CubeInstanceResponse.java | 26 +-
.../kylin/rest/response/CuboidTreeResponse.java | 123 ++++
.../apache/kylin/rest/response/SQLResponse.java | 46 +-
.../rest/security/AclPermissionFactory.java | 11 +-
.../rest/security/AuthoritiesPopulator.java | 1 +
.../rest/security/ExternalAclProvider.java | 16 +-
.../apache/kylin/rest/service/BasicService.java | 4 +
.../apache/kylin/rest/service/CubeService.java | 115 +++-
.../kylin/rest/service/DashboardService.java | 333 ++++++++++
.../apache/kylin/rest/service/JobService.java | 434 +++++++++++-
.../apache/kylin/rest/service/ModelService.java | 2 +-
.../apache/kylin/rest/service/QueryService.java | 182 ++---
.../apache/kylin/rest/service/TableService.java | 1 -
.../kylin/rest/util/QueryRequestLimits.java | 121 ++++
.../apache/kylin/rest/util/ValidateUtil.java | 1 +
.../kylin/rest/util/QueryRequestLimitsTest.java | 69 ++
server/src/main/resources/kylinMetrics.xml | 79 +++
server/src/main/resources/kylinSecurity.xml | 5 +-
server/src/main/webapp/WEB-INF/web.xml | 1 +
.../rest/controller/CubeControllerTest.java | 9 -
.../kylin/rest/metrics/QueryMetricsTest.java | 42 ++
.../rest/security/QueryWithTableACLTest.java | 22 +-
.../kylin/rest/service/AdminServiceTest.java | 36 +-
.../kylin/rest/service/CacheServiceTest.java | 2 +-
.../kylin/rest/service/ServiceTestBase.java | 3 +-
.../source/hive/CreateFlatHiveTableStep.java | 2 +-
.../apache/kylin/source/hive/HiveMRInput.java | 9 +-
.../cardinality/HiveColumnCardinalityJob.java | 2 +-
.../hive/exception/SegmentEmptyException.java | 44 ++
.../org/apache/kylin/source/jdbc/CmdStep.java | 2 +-
.../apache/kylin/source/jdbc/HiveCmdStep.java | 2 +-
.../apache/kylin/source/jdbc/JdbcExplorer.java | 2 +-
.../kylin/source/jdbc/JdbcHiveMRInput.java | 17 +-
.../org/apache/kylin/source/jdbc/SqlUtil.java | 2 +-
.../apache/kylin/source/jdbc/SqlUtilTest.java | 6 +-
.../apache/kylin/source/kafka/KafkaMRInput.java | 2 +-
.../source/kafka/config/KafkaClusterConfig.java | 19 -
.../kylin/source/kafka/job/MergeOffsetStep.java | 4 +-
.../source/kafka/job/UpdateTimeRangeStep.java | 2 +-
.../kylin/storage/hbase/HBaseConnection.java | 8 +
.../hbase/cube/v2/CubeHBaseEndpointRPC.java | 22 +-
.../storage/hbase/cube/v2/CubeHBaseRPC.java | 12 +-
.../hbase/cube/v2/ExpectedSizeIterator.java | 4 +-
.../coprocessor/endpoint/CubeVisitService.java | 40 +-
.../endpoint/generated/CubeVisitProtos.java | 115 +++-
.../endpoint/protobuf/CubeVisit.proto | 1 +
.../kylin/storage/hbase/steps/BulkLoadJob.java | 2 +-
.../storage/hbase/steps/CreateHTableJob.java | 30 +-
.../kylin/storage/hbase/steps/CubeHFileJob.java | 30 +-
.../storage/hbase/steps/CubeHFileMapper.java | 14 +-
.../storage/hbase/steps/DeprecatedGCStep.java | 2 +-
.../storage/hbase/steps/HBaseCuboidWriter.java | 5 +-
.../hbase/steps/HBaseMROutput2Transition.java | 50 +-
.../kylin/storage/hbase/steps/HBaseMRSteps.java | 81 ++-
.../steps/HDFSPathGarbageCollectionStep.java | 2 +-
.../kylin/storage/hbase/steps/MergeGCStep.java | 3 +-
.../storage/hbase/steps/RowKeyWritable.java | 86 +++
.../hbase/util/DeployCoprocessorCLI.java | 150 +++--
.../hbase/util/HtableAlterMetadataCLI.java | 21 +-
.../storage/hbase/util/ZookeeperAclBuilder.java | 119 ++++
.../hbase/util/ZookeeperDistributedLock.java | 3 +-
.../hbase/steps/CubeHFileMapper2Test.java | 4 -
.../hbase/steps/CubeHFileMapperTest.java | 8 +-
.../hbase/util/ZookeeperAclBuilderTest.java | 97 +++
tool/pom.xml | 5 +
.../apache/kylin/tool/job/CubeBuildingCLI.java | 138 ++++
.../metrics/systemcube/CubeDescCreator.java | 656 +++++++++++++++++++
.../metrics/systemcube/CubeInstanceCreator.java | 88 +++
.../metrics/systemcube/HiveTableCreator.java | 278 ++++++++
.../metrics/systemcube/KylinTableCreator.java | 114 ++++
.../tool/metrics/systemcube/ModelCreator.java | 267 ++++++++
.../tool/metrics/systemcube/ProjectCreator.java | 101 +++
.../tool/metrics/systemcube/SCCreator.java | 262 ++++++++
.../metrics/systemcube/util/HiveSinkTool.java | 61 ++
tool/src/main/resources/SCSinkTools.json | 15 +
.../tool/metrics/systemcube/SCCreatorTest.java | 134 ++++
tool/src/test/resources/SCSinkTools.json | 1 +
webapp/app/index.html | 15 +-
webapp/app/js/app.js | 2 +-
webapp/app/js/controllers/access.js | 10 +-
webapp/app/js/controllers/admin.js | 5 +
webapp/app/js/controllers/badQuery.js | 2 +-
webapp/app/js/controllers/cube.js | 188 +++++-
webapp/app/js/controllers/cubeAdvanceSetting.js | 60 ++
webapp/app/js/controllers/cubeEdit.js | 13 +-
webapp/app/js/controllers/cubeMeasures.js | 10 +-
webapp/app/js/controllers/cubeSchema.js | 48 +-
webapp/app/js/controllers/dashboard.js | 296 +++++++++
webapp/app/js/controllers/job.js | 20 +-
webapp/app/js/controllers/modelSchema.js | 4 +-
webapp/app/js/controllers/page.js | 12 +-
webapp/app/js/controllers/query.js | 119 +++-
webapp/app/js/controllers/streamingConfig.js | 43 +-
webapp/app/js/directives/directives.js | 97 +++
webapp/app/js/directives/ui-grid.js | 12 +-
webapp/app/js/factories/graph.js | 77 ---
webapp/app/js/filters/filter.js | 21 +
webapp/app/js/model/cubeConfig.js | 80 ++-
webapp/app/js/model/cubeDescModel.js | 5 +-
webapp/app/js/model/dashboardConfig.js | 96 +++
webapp/app/js/model/jobListModel.js | 13 +
webapp/app/js/model/queryConfig.js | 139 ++++
webapp/app/js/services/cubes.js | 47 +-
webapp/app/js/services/dashboard.js | 95 +++
webapp/app/js/services/graph.js | 54 --
webapp/app/js/services/kylinProperties.js | 22 +-
webapp/app/js/services/query.js | 13 +-
webapp/app/js/utils/utils.js | 10 +
webapp/app/less/app.less | 88 +++
webapp/app/less/component.less | 4 +
webapp/app/partials/admin/admin.html | 44 +-
.../cubeDesigner/advanced_settings.html | 91 ++-
webapp/app/partials/cubeDesigner/measures.html | 6 +-
.../partials/cubeDesigner/refresh_settings.html | 34 +-
.../partials/cubeDesigner/streamingConfig.html | 27 +-
webapp/app/partials/cubes/cube_detail.html | 83 ++-
webapp/app/partials/cubes/cubes.html | 6 +-
webapp/app/partials/dashboard/dashboard.html | 151 +++++
webapp/app/partials/header.html | 3 +
webapp/app/partials/jobs/badQuery.html | 6 +-
webapp/app/partials/jobs/jobList.html | 4 +-
.../app/partials/modelDesigner/data_model.html | 2 +-
webapp/app/partials/models/model_detail.html | 2 +-
webapp/app/partials/models/models_tree.html | 4 +-
.../partials/projects/project_table_tree.html | 2 +-
webapp/app/partials/query/query_detail.html | 104 +--
.../app/partials/tables/source_table_tree.html | 2 +-
webapp/app/partials/tables/table_detail.html | 2 +-
webapp/app/partials/tables/table_load.html | 2 +-
webapp/app/routes.json | 8 +
webapp/bower.json | 14 +-
webapp/grunt.json | 9 +-
388 files changed, 18016 insertions(+), 3356 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/4d50b269/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --cc core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 277ee31,7763457..2effdfb
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@@ -1043,11 -1035,33 +1091,33 @@@ abstract public class KylinConfigBase i
return Integer.parseInt(getOptional("kylin.engine.mr.mapper-input-rows", "1000000"));
}
+ public int getCuboidStatsCalculatorMaxNumber() {
+ // set 1 to disable multi-thread statistics calculation
+ return Integer.parseInt(getOptional("kylin.engine.mr.max-cuboid-stats-calculator-number", "1"));
+ }
+
+ public int getCuboidNumberPerStatsCalculator() {
+ return Integer.parseInt(getOptional("kylin.engine.mr.cuboid-number-per-stats-calculator", "100"));
+ }
+
+ public int getHadoopJobPerReducerHLLCuboidNumber() {
+ return Integer.parseInt(getOptional("kylin.engine.mr.per-reducer-hll-cuboid-number", "100"));
+ }
+
+ public int getHadoopJobHLLMaxReducerNumber() {
+ // by default multi-reducer hll calculation is disabled
+ return Integer.parseInt(getOptional("kylin.engine.mr.hll-max-reducer-number", "1"));
+ }
+
//UHC: ultra high cardinality columns, contain the ShardByColumns and the GlobalDictionaryColumns
public int getUHCReducerCount() {
- return Integer.parseInt(getOptional("kylin.engine.mr.uhc-reducer-count", "5"));
+ return Integer.parseInt(getOptional("kylin.engine.mr.uhc-reducer-count", "1"));
}
+ public boolean isBuildUHCDictWithMREnabled() {
- return Boolean.parseBoolean(getOptional("kylin.engine.mr.build-uhc-dict", "true"));
++ return Boolean.parseBoolean(getOptional("kylin.engine.mr.build-uhc-dict-in-additional-step", "false"));
+ }
+
public boolean isBuildDictInReducerEnabled() {
return Boolean.parseBoolean(getOptional("kylin.engine.mr.build-dict-in-reducer", "true"));
}
@@@ -1383,14 -1354,6 +1458,22 @@@
return Boolean.parseBoolean(getOptional("kylin.web.cross-domain-enabled", "true"));
}
+ public boolean isAdminUserExportAllowed() {
+ return Boolean.parseBoolean(getOptional("kylin.web.export-allow-admin", "true"));
+ }
+
+ public boolean isNoneAdminUserExportAllowed() {
+ return Boolean.parseBoolean(getOptional("kylin.web.export-allow-other", "true"));
+ }
+
++ public String getPropertiesWhiteList() {
++ return getOptional("kylin.web.properties.whitelist",
++ "kylin.web.timezone,kylin.query.cache-enabled,kylin.env,kylin.web.hive-limit,kylin.storage.default,kylin.engine.default,kylin.web.link-hadoop,kylin.web.link-diagnostic,"
++ + "kylin.web.contact-mail,kylin.web.help.length,kylin.web.help.0,kylin.web.help.1,kylin.web.help.2,kylin.web.help.3,"
++ + "kylin.web.help,kylin.web.hide-measures,kylin.web.link-streaming-guide,kylin.server.external-acl-provider,kylin.security.profile,"
++ + "kylin.htrace.show-gui-trace-toggle,kylin.web.export-allow-admin,kylin.web.export-allow-other");
++ }
++
// ============================================================================
// RESTCLIENT
// ============================================================================
@@@ -1403,9 -1366,9 +1486,10 @@@
return Integer.valueOf(this.getOptional("kylin.restclient.connection.max-total", "200"));
}
- /**
- * metric
- */
+ // ============================================================================
+ // Metrics
+ // ============================================================================
++
public String getCoadhaleMetricsReportClassesNames() {
return getOptional("kylin.metrics.reporter-classes",
"org.apache.kylin.common.metrics.metrics2.JsonFileMetricsReporter,org.apache.kylin.common.metrics.metrics2.JmxMetricsReporter");
@@@ -1423,19 -1386,56 +1507,64 @@@
return getOptional("kylin.metrics.perflogger-class", "org.apache.kylin.common.metrics.perflog.PerfLogger");
}
+ public boolean isShowingGuiTraceToggle() {
+ return Boolean.valueOf(getOptional("kylin.htrace.show-gui-trace-toggle", "false"));
+ }
+
+ public boolean isHtraceTracingEveryQuery() {
+ return Boolean.valueOf(getOptional("kylin.htrace.trace-every-query", "false"));
+ }
+
- public String getPropertiesWhiteList() {
- return getOptional("kylin.web.properties.whitelist",
- "kylin.web.timezone,kylin.query.cache-enabled,kylin.env,kylin.web.hive-limit,kylin.storage.default,kylin.engine.default,kylin.web.link-hadoop,kylin.web.link-diagnostic,"
- + "kylin.web.contact-mail,kylin.web.help.length,kylin.web.help.0,kylin.web.help.1,kylin.web.help.2,kylin.web.help.3,"
- + "kylin.web.help,kylin.web.hide-measures,kylin.web.link-streaming-guide,kylin.server.external-acl-provider,kylin.security.profile,"
- + "kylin.htrace.show-gui-trace-toggle,kylin.web.export-allow-admin,kylin.web.export-allow-other");
+ public boolean isKylinMetricsMonitorEnabled() {
+ return Boolean.parseBoolean(getOptional("kylin.core.metrics.monitor-enabled", "false"));
+ }
+
+ public boolean isKylinMetricsReporterForQueryEnabled() {
+ return Boolean.parseBoolean(getOptional("kylin.core.metrics.reporter-query-enabled", "false"));
+ }
+
+ public boolean isKylinMetricsReporterForJobEnabled() {
+ return Boolean.parseBoolean(getOptional("kylin.core.metrics.reporter-job-enabled", "false"));
+ }
+
+ public String getKylinMetricsPrefix() {
+ return getOptional("kylin.core.metrics.prefix", "KYLIN").toUpperCase();
+ }
+
+ public String getKylinMetricsActiveReservoirDefaultClass() {
+ return getOptional("kylin.core.metrics.active-reservoir-default-class",
+ "org.apache.kylin.metrics.lib.impl.StubReservoir");
+ }
+
+ public String getKylinSystemCubeSinkDefaultClass() {
+ return getOptional("kylin.core.metrics.system-cube-sink-default-class",
+ "org.apache.kylin.metrics.lib.impl.hive.HiveSink");
+ }
+
+ public String getKylinMetricsSubjectSuffix() {
+ return getOptional("kylin.core.metric.subject-suffix", getDeployEnv());
+ }
+
+ public String getKylinMetricsSubjectJob() {
+ return getOptional("kylin.core.metrics.subject-job", "METRICS_JOB") + "_" + getKylinMetricsSubjectSuffix();
+ }
+
+ public String getKylinMetricsSubjectJobException() {
+ return getOptional("kylin.core.metrics.subject-job-exception", "METRICS_JOB_EXCEPTION") + "_"
+ + getKylinMetricsSubjectSuffix();
+ }
+
+ public String getKylinMetricsSubjectQuery() {
+ return getOptional("kylin.core.metrics.subject-query", "METRICS_QUERY") + "_" + getKylinMetricsSubjectSuffix();
+ }
+
+ public String getKylinMetricsSubjectQueryCube() {
+ return getOptional("kylin.core.metrics.subject-query-cube", "METRICS_QUERY_CUBE") + "_"
+ + getKylinMetricsSubjectSuffix();
+ }
+
+ public String getKylinMetricsSubjectQueryRpcCall() {
+ return getOptional("kylin.core.metrics.subject-query-rpc", "METRICS_QUERY_RPC") + "_"
+ + getKylinMetricsSubjectSuffix();
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/4d50b269/core-common/src/main/java/org/apache/kylin/common/QueryContext.java
----------------------------------------------------------------------
diff --cc core-common/src/main/java/org/apache/kylin/common/QueryContext.java
index f00a7cc,c7ded10..1af90f4
--- a/core-common/src/main/java/org/apache/kylin/common/QueryContext.java
+++ b/core-common/src/main/java/org/apache/kylin/common/QueryContext.java
@@@ -18,7 -18,9 +18,10 @@@
package org.apache.kylin.common;
+ import java.io.Serializable;
+ import java.util.List;
+ import java.util.Map;
+import java.util.Set;
import java.util.UUID;
import java.util.concurrent.atomic.AtomicLong;
@@@ -34,9 -45,11 +46,12 @@@ public class QueryContext
}
};
+ private long queryStartMillis;
+ private long deadline = Long.MAX_VALUE;
+
private String queryId;
private String username;
+ private Set<String> groups;
private AtomicLong scannedRows = new AtomicLong();
private AtomicLong scannedBytes = new AtomicLong();
http://git-wip-us.apache.org/repos/asf/kylin/blob/4d50b269/core-common/src/main/java/org/apache/kylin/common/util/BytesUtil.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/4d50b269/core-common/src/main/java/org/apache/kylin/common/util/HadoopUtil.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/4d50b269/core-common/src/main/resources/kylin-defaults.properties
----------------------------------------------------------------------
diff --cc core-common/src/main/resources/kylin-defaults.properties
index 51e5a2d,6ca516d..e87b31d
--- a/core-common/src/main/resources/kylin-defaults.properties
+++ b/core-common/src/main/resources/kylin-defaults.properties
@@@ -43,12 -43,14 +43,18 @@@ kylin.server.cluster-servers=localhost:
# Display timezone on UI,format like[GMT+N or GMT-N]
kylin.web.timezone=GMT+8
+ # Timeout value for the queries submitted through the Web UI, in milliseconds
+ kylin.web.query-timeout=300000
+
kylin.web.cross-domain-enabled=true
+#allow user to export query result
+kylin.web.export-allow-admin=true
+kylin.web.export-allow-other=true
+
+ # Hide measures in measure list of cube designer, separate by comma
+ kylin.web.hide-measures=RAW
+
#max connections of one route
kylin.restclient.connection.default-max-per-route=20
@@@ -179,6 -166,6 +188,9 @@@ kylin.engine.mr.build-dict-in-reducer=t
# Number of reducers for fetching UHC column distinct values
kylin.engine.mr.uhc-reducer-count=1
++# Whether using an additional step to build UHC dictionary
++kylin.engine.mr.build-uhc-dict-in-additional-step=false
++
### CUBE | DICTIONARY ###
http://git-wip-us.apache.org/repos/asf/kylin/blob/4d50b269/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java
----------------------------------------------------------------------
diff --cc core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java
index a416499,d522263..cd43f15
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java
@@@ -25,10 -25,12 +25,12 @@@ import java.util.Map
import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
+ import org.apache.kylin.common.persistence.JsonSerializer;
import org.apache.kylin.common.persistence.ResourceStore;
+ import org.apache.kylin.common.persistence.Serializer;
-import org.apache.kylin.cube.cuboid.Cuboid;
+import org.apache.kylin.common.util.AutoReadWriteLock;
+import org.apache.kylin.common.util.AutoReadWriteLock.AutoLock;
+import org.apache.kylin.cube.cuboid.CuboidManager;
import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.cube.model.validation.CubeMetadataValidator;
import org.apache.kylin.cube.model.validation.ValidateContext;
@@@ -58,15 -61,39 +60,17 @@@ public class CubeDescManager
private static final Logger logger = LoggerFactory.getLogger(CubeDescManager.class);
+ public static final Serializer<CubeDesc> CUBE_DESC_SERIALIZER = new JsonSerializer<CubeDesc>(CubeDesc.class);
-
- // static cached instances
- private static final ConcurrentMap<KylinConfig, CubeDescManager> CACHE = new ConcurrentHashMap<KylinConfig, CubeDescManager>();
-
++
public static CubeDescManager getInstance(KylinConfig config) {
- CubeDescManager r = CACHE.get(config);
- if (r != null) {
- return r;
- }
-
- synchronized (CubeDescManager.class) {
- r = CACHE.get(config);
- if (r != null) {
- return r;
- }
- try {
- r = new CubeDescManager(config);
- CACHE.put(config, r);
- if (CACHE.size() > 1) {
- logger.warn("More than one singleton exist");
- }
- return r;
- } catch (IOException e) {
- throw new IllegalStateException("Failed to init CubeDescManager from " + config, e);
- }
- }
+ return config.getManager(CubeDescManager.class);
}
- public static void clearCache() {
- CACHE.clear();
+ // called by reflection
+ static CubeDescManager newInstance(KylinConfig config) throws IOException {
+ return new CubeDescManager(config);
}
-
+
// ============================================================================
private KylinConfig config;
http://git-wip-us.apache.org/repos/asf/kylin/blob/4d50b269/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
----------------------------------------------------------------------
diff --cc core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
index 55e9325,70477eb..79341cb
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
@@@ -18,7 -18,8 +18,9 @@@
package org.apache.kylin.cube;
+import static com.google.common.base.Preconditions.checkNotNull;
+ import static org.apache.kylin.cube.cuboid.CuboidModeEnum.CURRENT;
+ import static org.apache.kylin.cube.cuboid.CuboidModeEnum.RECOMMEND;
import java.io.IOException;
import java.util.List;
@@@ -343,6 -332,6 +356,14 @@@ public class CubeInstance extends RootP
}
return null;
}
++
++ public CubeSegment[] regetSegments(CubeSegment... segs) {
++ CubeSegment[] r = new CubeSegment[segs.length];
++ for (int i = 0; i < segs.length; i++) {
++ r[i] = getSegmentById(segs[i].getUuid());
++ }
++ return r;
++ }
public long getCreateTimeUTC() {
return createTimeUTC;
@@@ -520,6 -546,6 +578,36 @@@
}
}
++ public long getSizeKB() {
++ long sizeKb = 0L;
++
++ for (CubeSegment cubeSegment : this.getSegments(SegmentStatusEnum.READY)) {
++ sizeKb += cubeSegment.getSizeKB();
++ }
++
++ return sizeKb;
++ }
++
++ public long getInputRecordCount() {
++ long sizeRecordCount = 0L;
++
++ for (CubeSegment cubeSegment : this.getSegments(SegmentStatusEnum.READY)) {
++ sizeRecordCount += cubeSegment.getInputRecords();
++ }
++
++ return sizeRecordCount;
++ }
++
++ public long getInputRecordSizeMB() {
++ long sizeRecordSize = 0L;
++
++ for (CubeSegment cubeSegment : this.getSegments(SegmentStatusEnum.READY)) {
++ sizeRecordSize += cubeSegment.getInputRecordsSize();
++ }
++
++ return sizeRecordSize;
++ }
++
public String getProject() {
return getDescriptor().getProject();
}