You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2019/03/26 14:20:47 UTC
[kylin] branch master updated (6ee0212 -> c137bc8)
This is an automated email from the ASF dual-hosted git repository.
shaofengshi pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git.
from 6ee0212 KYLIN-3895 Failed to register new MBean when "kylin.server.query-metrics-enabled" set true
add c05ae0c KYLIN-3691 New streaming ui implementation
add 0943599 KYLIN-3690 New streaming backend implementation
add dd4ea5b KYLIN-3690 New streaming backend implementation - streaming consumer
add 363749e KYLIN-3690 New streaming backend implementation - streaming job
add 6da9795 KYLIN-3730 TableMetadataManager.reloadSourceTableQuietly is wrong
add 4a1a4de KYLIN-3742 Fix DataRequest for NPE and add some javadoc
add 4e370b2 KYLIN-3745 Real-time segment state changed from active to immutable is not sequently
add b253b2c KYLIN-3747 Use FQDN to register a streaming receiver instead of ip
add 1516d8c KYLIN-3768 Save streaming metadata a standard kylin path in zookeeper
add 63268a0 KYLIN-3759 Fix classnotfound if lambda is enable
add 13b7612 KYLIN-3744 Make coordinator well tested and add some javadoc
add b7a52af KYLIN-3784 Hadoop Common ReflectionUtils.printThreadInfo signature change
add 2cbb8cd KYLIN-3787 NPE throws when dimension value has null when query real-time data
add bf446aa KYLIN-3791 Map return by Maps.transformValues is a immutable view
add 0669cee KYLIN-3789 Stream receiver admin page issue fix
add 6cd7061 KYLIN-3800 Measure get incorrect result
add 14be024 KYLIN-3786 Add integration test for realtime-streaming
add ad9b49b Minor, fix rat check failed
add 9a363c1 KYLIN-3744 Add configuration and fix bugs
add e70a6f7 KYLIN-3821 Add consume lag stats
add a774221 minor, less info log
add 80fbff4 KYLIN-3654 rebase master
add c137bc8 KYLIN-3768 Save streaming metadata a standard kylin path in zookeeper
No new revisions were added by this update.
Summary of changes:
assembly/pom.xml | 9 +
.../kylin/job/streaming/Kafka10DataLoader.java | 2 +-
build/bin/kylin.sh | 109 +
build/script/prepare-libs.sh | 3 +
.../org/apache/kylin/common/KylinConfigBase.java | 146 +
.../org/apache/kylin/common/KylinConfigExt.java | 2 +-
.../apache/kylin/common/debug/BackdoorToggles.java | 14 +
.../kylin/common/persistence/ResourceStore.java | 1 +
.../org/apache/kylin/common/util/ServerMode.java | 74 +
.../org/apache/kylin/common/util/TimeUtil.java | 36 +-
.../java/org/apache/kylin/common/util/ZKUtil.java | 16 +
.../java/org/apache/kylin/cube/CubeSegment.java | 33 +
.../java/org/apache/kylin/cube/model/CubeDesc.java | 4 +
.../org/apache/kylin/dict/DictionaryGenerator.java | 13 +-
.../dict/MultipleDictionaryValueEnumerator.java | 9 +-
.../MultipleDictionaryValueEnumeratorTest.java | 13 +-
.../kylin/job/constant/ExecutableConstants.java | 4 +
.../job/impl/threadpool/DefaultScheduler.java | 2 +-
.../apache/kylin/measure/topn/TopNAggregator.java | 8 +
.../org/apache/kylin/measure/topn/TopNCounter.java | 6 +
.../kylin/metadata/TableMetadataManager.java | 2 +-
.../apache/kylin/metadata/model/ISourceAware.java | 2 +
.../apache/kylin/metadata/model/IStorageAware.java | 1 +
.../apache/kylin/metadata/model/ParameterDesc.java | 14 +-
.../org/apache/kylin/metadata/model/TableDesc.java | 8 +
.../kylin/metadata/project/ProjectManager.java | 7 +
.../org/apache/kylin/storage/StorageFactory.java | 13 +
dev-support/checkstyle-suppressions.xml | 2 +
engine-mr/pom.xml | 11 +
.../java/org/apache/kylin/engine/mr/CubingJob.java | 6 +-
.../kylin/engine/mr/StreamingCubingEngine.java | 20 +-
.../kylin/engine/mr/StreamingCubingJobBuilder.java | 220 +
.../engine/mr/steps/CubingExecutableUtil.java | 9 +
.../mr/steps/UpdateCubeInfoAfterBuildStep.java | 3 +-
.../mr/steps/UpdateCubeInfoAfterMergeStep.java | 6 +
.../kylin/engine/mr/streaming/ColumnToRowJob.java | 133 +
.../engine/mr/streaming/ColumnToRowMapper.java | 29 +-
.../engine/mr/streaming/ColumnToRowReducer.java | 84 +
.../engine/mr/streaming/ColumnarFilesReader.java | 73 +
.../mr/streaming/ColumnarSplitDataInputFormat.java | 27 +-
.../mr/streaming/ColumnarSplitDataReader.java | 111 +
.../mr/streaming/ColumnarSplitDictInputFormat.java | 27 +-
.../mr/streaming/ColumnarSplitDictReader.java | 105 +
.../mr/streaming/ColumnarSplitInputFormat.java | 133 +
.../engine/mr/streaming/ColumnarSplitReader.java | 71 +
.../kylin/engine/mr/streaming/DictsReader.java | 76 +
.../kylin/engine/mr/streaming/MergeDictJob.java | 103 +
.../kylin/engine/mr/streaming/MergeDictMapper.java | 28 +-
.../engine/mr/streaming/MergeDictReducer.java | 141 +
.../kylin/engine/mr/streaming/RowRecord.java | 31 +-
.../kylin/engine/mr/streaming/RowRecordReader.java | 266 +
.../kylin/engine/mr/streaming/SaveDictStep.java | 155 +
.../localmeta/cube/test_streaming_v2_cube.json | 17 +
.../cube/test_streaming_v2_user_info_cube.json | 17 +
.../cube_desc/test_streaming_v2_cube.json | 139 +
.../test_streaming_v2_user_info_cube.json | 262 +
.../data/DEFAULT.STREAMING_V2_USER_INFO_TABLE.csv | 1 +
examples/test_case_data/localmeta/kylin.properties | 1 +
.../model_desc/test_streaming_v2_model.json | 30 +
.../test_streaming_v2_user_info_model.json | 27 +
.../test_case_data/localmeta/project/default.json | 18 +-
.../DEFAULT.STREAMING_V2_USER_INFO_TABLE.json | 15 +
.../table/DEFAULT.STREAMING_V2_TABLE.json | 47 +
.../DEFAULT.STREAMING_V2_USER_INFO_TABLE.json | 86 +
kylin-it/pom.xml | 36 +-
.../kylin/provision/BuildCubeWithStream.java | 26 +-
.../java/org/apache/kylin/provision/MockKafka.java | 19 +-
.../java/org/apache/kylin/query/H2Database.java | 3 +-
.../kylin/realtime/BuildCubeWithStreamV2.java | 477 +
.../sql_streaming_v2/compare_result/query01.sql | 19 +
.../sql_streaming_v2/compare_result/query02.sql | 19 +
.../sql_streaming_v2/compare_result/query03.sql | 19 +
.../sql_streaming_v2/compare_result/query04.sql | 19 +
.../sql_streaming_v2/compare_result/query05.sql | 19 +
.../sql_streaming_v2/compare_result/query06.sql | 19 +
.../sql_streaming_v2/compare_result/query07.sql | 19 +
.../sql_streaming_v2/compare_result/query08.sql | 19 +
.../query/sql_streaming_v2/count/query01.sql | 19 +
.../not_compare_result/query01.sql | 20 +
.../not_compare_result/query02.sql | 20 +
.../resources/streaming_v2_user_info_messages.txt | 10000 +++++++++++++++++++
pom.xml | 42 +-
server-base/pom.xml | 43 +
.../controller/StreamingCoordinatorController.java | 196 +
.../rest/controller/StreamingV2Controller.java | 524 +
.../kylin/rest/request/StreamingRequestV2.java | 73 +
.../org/apache/kylin/rest/service/CubeService.java | 53 +-
.../rest/service/StreamingCoordinatorService.java | 117 +
.../kylin/rest/service/StreamingV2Service.java | 586 ++
server/src/main/resources/kylinSecurity.xml | 2 +
.../kylin/storage/hbase/steps/CreateHTableJob.java | 8 +-
.../storage/hbase/util/DeployCoprocessorCLI.java | 2 +-
{engine-mr => storage-stream}/pom.xml | 93 +-
.../apache/kylin/storage/stream/StreamStorage.java | 72 +
.../kylin/storage/stream/StreamStorageQuery.java | 130 +
.../stream/rpc/HttpStreamDataSearchClient.java | 327 +
.../stream/rpc/IStreamDataSearchClient.java | 45 +
.../stream/rpc/MockedStreamDataSearchClient.java | 40 +
{engine-mr => stream-coordinator}/pom.xml | 100 +-
.../kylin/stream/coordinator/Coordinator.java | 1465 +++
.../coordinator/HBaseStreamMetadataStore.java | 178 +
.../coordinator/MockStreamMetadataStore.java | 178 +
.../stream/coordinator/StreamMetadataStore.java | 117 +
.../coordinator/StreamMetadataStoreFactory.java | 61 +
.../stream/coordinator/StreamingCubeInfo.java | 66 +
.../kylin/stream/coordinator/StreamingUtils.java | 25 +-
.../coordinator/ZookeeperStreamMetadataStore.java | 578 ++
.../kylin/stream/coordinator/assign/Assigner.java | 49 +
.../stream/coordinator/assign/AssignmentUtil.java | 87 +
.../coordinator/assign/AssignmentsCache.java | 83 +
.../assign/CubePartitionRoundRobinAssigner.java | 322 +
.../stream/coordinator/assign/DefaultAssigner.java | 210 +
.../coordinator/client/CoordinatorClient.java | 56 +
.../client/CoordinatorClientFactory.java | 55 +
.../coordinator/client/CoordinatorResponse.java | 58 +
.../coordinator/client/HttpCoordinatorClient.java | 298 +
.../exception/ClusterStateException.java | 59 +
.../coordinator/exception/CoordinateException.java | 30 +-
.../exception/NotLeadCoordinatorException.java | 31 +-
.../coordinator/exception/StoreException.java | 24 +-
.../kylin/stream/coordinator/CoordinatorTest.java | 278 +
.../CubePartitionRoundRobinAssignerTest.java | 262 +
.../coordinator/assign/DefaultAssignerTest.java | 183 +
{engine-mr => stream-core}/pom.xml | 91 +-
.../core/client/HttpReceiverAdminClient.java | 255 +
.../stream/core/client/ReceiverAdminClient.java | 63 +
.../stream/core/consumer/ConsumerStartMode.java | 18 +-
.../core/consumer/ConsumerStartProtocol.java | 92 +
.../core/consumer/EndPositionStopCondition.java | 86 +
.../stream/core/consumer/IConsumerProvider.java | 15 +-
.../core/consumer/IStopConsumptionCondition.java | 21 +-
.../stream/core/consumer/IStreamingConnector.java | 33 +-
.../consumer/NeverStopConsumptionCondition.java | 26 +-
.../core/consumer/StreamingConsumerChannel.java | 299 +
.../core/exception/IllegalStorageException.java | 27 +-
.../stream/core/exception/StreamingException.java | 26 +-
.../stream/core/metrics/StreamingMetrics.java | 51 +
.../kylin/stream/core/model/AssignRequest.java | 59 +
.../stream/core/model/ConsumerStatsResponse.java | 31 +-
.../kylin/stream/core/model/CubeAssignment.java | 147 +
.../kylin/stream/core/model/DataRequest.java | 145 +
.../kylin/stream/core/model/DataResponse.java | 32 +-
.../kylin/stream/core/model/HealthCheckInfo.java | 23 +-
.../org/apache/kylin/stream/core/model/Node.java | 110 +
.../stream/core/model/PauseConsumersRequest.java | 24 +-
.../core/model/RemoteStoreCompleteRequest.java | 58 +
.../apache/kylin/stream/core/model/ReplicaSet.java | 104 +
.../core/model/ReplicaSetLeaderChangeRequest.java | 36 +-
.../stream/core/model/ResumeConsumerRequest.java | 36 +-
.../kylin/stream/core/model/SegmentBuildState.java | 144 +
.../stream/core/model/StartConsumersRequest.java | 38 +-
.../stream/core/model/StopConsumersRequest.java | 36 +-
.../core/model/StreamingCubeConsumeState.java | 15 +-
.../kylin/stream/core/model/StreamingMessage.java | 79 +
.../kylin/stream/core/model/UnAssignRequest.java | 24 +-
.../stream/core/model/stats/ClusterState.java | 75 +
.../stream/core/model/stats/ConsumerStats.java | 105 +
.../stream/core/model/stats/CubeRealTimeState.java | 52 +
.../stream/core/model/stats/LongLatencyInfo.java | 85 +
.../core/model/stats/PartitionConsumeStats.java | 86 +
.../model/stats/ReceiverCubeRealTimeState.java | 47 +
.../stream/core/model/stats/ReceiverCubeStats.java | 100 +
.../stream/core/model/stats/ReceiverState.java | 82 +
.../stream/core/model/stats/ReceiverStats.java | 80 +
.../stream/core/model/stats/ReplicaSetState.java | 92 +
.../stream/core/model/stats/SegmentStats.java | 91 +
.../stream/core/model/stats/SegmentStoreStats.java | 47 +
.../stream/core/query/HavingFilterChecker.java | 135 +
.../stream/core/query/IStreamingGTSearcher.java | 15 +-
.../stream/core/query/IStreamingSearchResult.java | 59 +
.../core/query/MultiThreadsResultCollector.java | 161 +
.../kylin/stream/core/query/RecordsAggregator.java | 160 +
.../stream/core/query/ResponseResultSchema.java | 157 +
.../kylin/stream/core/query/ResultCollector.java | 76 +
.../core/query/SingleThreadResultCollector.java | 75 +
.../query/StreamingBuiltInFunctionTransformer.java | 272 +
.../core/query/StreamingCubeDataSearcher.java | 196 +
.../core/query/StreamingDataQueryPlanner.java | 108 +
.../stream/core/query/StreamingQueryProfile.java | 194 +
.../stream/core/query/StreamingSearchContext.java | 161 +
.../stream/core/query/StreamingTupleConverter.java | 121 +
.../stream/core/query/StreamingTupleIterator.java | 126 +
.../kylin/stream/core/source/ISourcePosition.java | 37 +-
.../stream/core/source/ISourcePositionHandler.java | 29 +-
.../core/source/IStreamingMessageParser.java | 15 +-
.../kylin/stream/core/source/IStreamingSource.java | 61 +
.../stream/core/source/MessageFormatException.java | 25 +-
.../stream/core/source/MessageParserInfo.java | 61 +
.../apache/kylin/stream/core/source/Partition.java | 96 +
.../stream/core/source/StreamingSourceConfig.java | 99 +
.../core/source/StreamingSourceConfigManager.java | 185 +
.../stream/core/source/StreamingSourceFactory.java | 44 +
.../core/source/StreamingTableSourceInfo.java | 40 +-
.../kylin/stream/core/storage/CheckPoint.java | 135 +
.../kylin/stream/core/storage/CheckPointStore.java | 225 +
.../core/storage/IStreamingSegmentStore.java | 57 +
.../apache/kylin/stream/core/storage/Record.java | 68 +
.../stream/core/storage/StreamingCubeSegment.java | 187 +
.../core/storage/StreamingSegmentManager.java | 628 ++
.../core/storage/columnar/ColumnDataReader.java | 16 +-
.../core/storage/columnar/ColumnDataWriter.java | 15 +-
.../columnar/ColumnarMemoryStorePersister.java | 406 +
.../storage/columnar/ColumnarMetricsEncoding.java | 28 +-
.../columnar/ColumnarMetricsEncodingFactory.java | 181 +
.../core/storage/columnar/ColumnarRecordCodec.java | 105 +
.../storage/columnar/ColumnarSegmentStore.java | 465 +
.../ColumnarSegmentStoreFilesSearcher.java | 104 +
.../core/storage/columnar/ColumnarStoreCache.java | 172 +
.../storage/columnar/ColumnarStoreCacheStats.java | 95 +
.../storage/columnar/ColumnarStoreDimDesc.java | 93 +
.../storage/columnar/ColumnarStoreMetricsDesc.java | 87 +
.../core/storage/columnar/DataSegmentFragment.java | 159 +
.../columnar/FSInputGeneralColumnDataReader.java | 77 +
.../storage/columnar/FragmentCuboidReader.java | 179 +
.../stream/core/storage/columnar/FragmentData.java | 111 +
.../storage/columnar/FragmentFileSearcher.java | 360 +
.../core/storage/columnar/FragmentFilesMerger.java | 624 ++
.../stream/core/storage/columnar/FragmentId.java | 89 +
.../storage/columnar/FragmentSearchResult.java | 483 +
.../storage/columnar/FragmentsMergeResult.java | 53 +
.../storage/columnar/GeneralColumnDataReader.java | 106 +
.../storage/columnar/GeneralColumnDataWriter.java | 60 +
.../storage/columnar/ParsedStreamingCubeInfo.java | 263 +
.../stream/core/storage/columnar/RawRecord.java | 77 +
.../core/storage/columnar/SegmentMemoryStore.java | 436 +
.../storage/columnar/StringArrayComparator.java | 50 +
.../columnar/TimeDerivedColumnEncoding.java | 42 +
.../storage/columnar/compress/Compression.java | 15 +-
.../compress/FSInputLZ4CompressedColumnReader.java | 115 +
.../compress/FSInputNoCompressedColumnReader.java | 83 +
.../compress/FSInputRLECompressedColumnReader.java | 124 +
.../compress/LZ4CompressedColumnReader.java | 125 +
.../compress/LZ4CompressedColumnWriter.java | 78 +
.../columnar/compress/LZ4CompressorTest.java | 66 +
.../compress/NoCompressedColumnReader.java | 82 +
.../compress/NoCompressedColumnWriter.java | 31 +-
.../compress/RunLengthCompressedColumnReader.java | 153 +
.../compress/RunLengthCompressedColumnWriter.java | 130 +
.../invertindex/ColInvertIndexSearcher.java | 169 +
.../columnar/invertindex/ColInvertIndexWriter.java | 34 +-
.../invertindex/FixLenColInvertIndexWriter.java | 87 +
.../invertindex/FixLenIIColumnDescriptor.java | 28 +-
.../columnar/invertindex/IIColumnDescriptor.java | 21 +-
.../columnar/invertindex/IndexSearchResult.java | 18 +-
.../columnar/invertindex/InvertIndexSearcher.java | 341 +
.../invertindex/SeqColInvertIndexWriter.java | 111 +
.../invertindex/SeqIIColumnDescriptor.java | 28 +-
.../storage/columnar/protocol/CuboidMetaInfo.java | 114 +
.../columnar/protocol/DimDictionaryMetaInfo.java | 93 +
.../columnar/protocol/DimensionMetaInfo.java | 140 +
.../core/storage/columnar/protocol/Footer.java | 87 +
.../columnar/protocol/FragmentMetaInfo.java | 171 +
.../storage/columnar/protocol/MetricMetaInfo.java | 142 +
.../core/storage/rocksdb/RocksDBSegmentStore.java | 136 +
.../core/util/CompareFilterTimeRangeChecker.java | 137 +
.../apache/kylin/stream/core/util/Constants.java | 16 +-
.../apache/kylin/stream/core/util/HDFSUtil.java | 86 +
.../kylin/stream/core/util/NamedThreadFactory.java | 36 +-
.../apache/kylin/stream/core/util/NodeUtil.java | 58 +
.../kylin/stream/core/util/RecordsSerializer.java | 126 +
.../apache/kylin/stream/core/util/RestService.java | 133 +
.../kylin/stream/core/util/RetryCallable.java | 23 +-
.../apache/kylin/stream/core/util/RetryCaller.java | 83 +
.../kylin/stream/core/util/StreamFilterUtil.java | 56 +
.../stream/core/util/TimeDerivedColumnType.java | 219 +
.../stream/core/storage/CheckPointStoreTest.java | 142 +
.../stream/core/storage/MockPositionHandler.java | 72 +
.../stream/core/storage/RecordsSerDeTest.java | 91 +
.../core/storage/StreamingSegmentManagerTest.java | 180 +
.../kylin/stream/core/storage/TestHelper.java | 158 +
.../storage/columnar/ColumnarSegmentStoreTest.java | 206 +
.../storage/columnar/FragmentCuboidReaderTest.java | 134 +
.../storage/columnar/FragmentFileSearcherTest.java | 208 +
.../storage/columnar/FragmentFilesMergerTest.java | 229 +
.../storage/columnar/GeneralColumnDataTest.java | 81 +
.../storage/columnar/SegmentMemoryStoreTest.java | 252 +
.../storage/columnar/StreamingDataSimulator.java | 134 +
.../columnar/compress/LZ4CompressColumnTest.java | 107 +
.../columnar/compress/NoCompressColumnTest.java | 99 +
.../compress/RunLengthCompressColumnTest.java | 168 +
.../ColInvertIndexWriterWriterTest.java | 79 +
.../SearchableColInvertIndexWriterTest.java | 24 +-
.../SimpleColInvertIndexWriterTest.java | 72 +
.../performance/FragmentCuboidReaderPerfTest.java | 203 +
.../columnar/performance/PerfDataPrepare.java | 86 +
.../columnar/performance/PerformanceTest.java | 163 +
.../StreamingCubeDataSearcherPerfTest.java | 185 +
.../kylin/stream/core/util/DataGenerator.java | 79 +
.../core/util/TimeDerivedColumnTypeTest.java | 118 +
stream-receiver/pom.xml | 308 +
.../stream/server/ReplicaSetLeaderSelector.java | 100 +
.../apache/kylin/stream/server/ServerContext.java | 34 +-
.../kylin/stream/server/StreamingReceiver.java | 147 +
.../kylin/stream/server/StreamingServer.java | 746 ++
.../server/rest/controller/AdminController.java | 165 +
.../server/rest/controller/BasicController.java | 69 +
.../server/rest/controller/DataController.java | 169 +
.../server/rest/controller/QueryController.java | 104 +
.../server/rest/controller/StatsController.java | 68 +
.../server/rest/controller/SystemController.java | 135 +
.../server/rest/exception/BadRequestException.java | 48 +-
.../server/rest/exception/ForbiddenException.java | 28 +-
.../rest/exception/InternalErrorException.java | 63 +
.../server/rest/exception/NotFoundException.java | 19 +-
.../stream/server/rest/model/ErrorResponse.java | 22 +-
.../server/rest/model/PrepareSqlRequest.java | 121 +
.../kylin/stream/server/rest/model/SQLRequest.java | 132 +
.../stream/server/rest/model/SQLResponse.java | 167 +
.../rest/security/StreamTableInterceptor.java | 64 +
.../stream/server/rest/service/BasicService.java | 55 +
.../stream/server/rest/service/QueryService.java | 274 +
.../kylin/stream/server/rest/util/QueryUtil.java | 225 +
.../server/retention/RetentionPolicyInfo.java | 60 +
.../stream/server/storage/LocalStreamStorage.java | 31 +-
.../server/storage/LocalStreamStorageQuery.java | 64 +
.../src/main/resources/applicationContext.xml | 77 +
.../src/main/resources/ehcache-test.xml | 37 +
stream-receiver/src/main/resources/ehcache.xml | 37 +
.../resources/stream-receiver-log4j.properties | 44 +
.../src/main/webapp/WEB-INF/kylin-servlet.xml | 19 +
stream-receiver/src/main/webapp/index.html | 23 +
stream-source-kafka/pom.xml | 74 +
.../stream/source/kafka/BootstrapServerConfig.java | 54 +
.../source/kafka/KafkaBatchSourceAdaptor.java | 43 +
.../source/kafka/KafkaConsumerStartInfo.java | 34 +-
.../kylin/stream/source/kafka/KafkaPosition.java | 111 +
.../stream/source/kafka/KafkaPositionHandler.java | 86 +
.../kylin/stream/source/kafka/KafkaSource.java | 329 +
.../stream/source/kafka/KafkaTopicAssignment.java | 64 +
.../stream/source/kafka/TimedJsonStreamParser.java | 121 +
.../source/kafka/consumer/KafkaConnector.java | 125 +
.../assign/KafkaSourcePositionHandlerTest.java | 70 +
.../org/apache/kylin/tool/CubeMigrationCLI.java | 6 +
.../apache/kylin/tool/CubeMetaExtractorTest.java | 4 +-
webapp/app/index.html | 5 +
webapp/app/js/app.js | 2 +-
webapp/app/js/controllers/adminStreaming.js | 330 +
webapp/app/js/controllers/cube.js | 36 +-
webapp/app/js/controllers/cubeOverwriteProp.js | 59 +-
webapp/app/js/controllers/cubeSchema.js | 10 +-
webapp/app/js/controllers/cubes.js | 566 +-
webapp/app/js/controllers/sourceMeta.js | 448 +-
.../js/controllers/streamingBalanceAssignGroup.js | 84 +
webapp/app/js/directives/directives.js | 2 +-
webapp/app/js/model/tableConfig.js | 6 +-
webapp/app/js/model/tableModel.js | 2 +-
webapp/app/js/services/streaming.js | 269 +
webapp/app/js/utils/liquidFillGauge.js | 268 +
webapp/app/less/app.less | 73 +
webapp/app/partials/admin/admin.html | 122 +-
.../app/partials/admin/{admin.html => config.html} | 0
webapp/app/partials/admin/streaming.html | 146 +
webapp/app/partials/admin/streamingReceiver.html | 177 +
.../partials/cubeDesigner/cubeOverwriteProp.html | 82 +-
webapp/app/partials/cubes/cube_detail.html | 116 +
webapp/app/partials/cubes/cubes.html | 22 +-
.../app/partials/streaming/balanceReplicaSet.html | 72 +
webapp/app/partials/streaming/cubeAssignment.html | 60 +
.../partials/tables/loadStreamingSourceConfig.html | 79 +
.../partials/tables/loadStreamingTableConfig.html | 140 +
webapp/app/partials/tables/source_table_tree.html | 1 +
webapp/app/partials/tables/table_detail.html | 29 +
webapp/app/partials/tables/table_load.html | 6 +
webapp/app/routes.json | 8 +
webapp/bower.json | 3 +-
webapp/grunt.json | 2 +
366 files changed, 47038 insertions(+), 1117 deletions(-)
create mode 100644 core-common/src/main/java/org/apache/kylin/common/util/ServerMode.java
copy core-metadata/src/main/java/org/apache/kylin/metadata/model/IStorageAware.java => engine-mr/src/main/java/org/apache/kylin/engine/mr/StreamingCubingEngine.java (66%)
create mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/StreamingCubingJobBuilder.java
create mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/streaming/ColumnToRowJob.java
copy core-metadata/src/main/java/org/apache/kylin/metadata/model/ISourceAware.java => engine-mr/src/main/java/org/apache/kylin/engine/mr/streaming/ColumnToRowMapper.java (58%)
create mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/streaming/ColumnToRowReducer.java
create mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/streaming/ColumnarFilesReader.java
copy core-metadata/src/main/java/org/apache/kylin/metadata/model/ISourceAware.java => engine-mr/src/main/java/org/apache/kylin/engine/mr/streaming/ColumnarSplitDataInputFormat.java (60%)
create mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/streaming/ColumnarSplitDataReader.java
copy core-metadata/src/main/java/org/apache/kylin/metadata/model/ISourceAware.java => engine-mr/src/main/java/org/apache/kylin/engine/mr/streaming/ColumnarSplitDictInputFormat.java (60%)
create mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/streaming/ColumnarSplitDictReader.java
create mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/streaming/ColumnarSplitInputFormat.java
create mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/streaming/ColumnarSplitReader.java
create mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/streaming/DictsReader.java
create mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/streaming/MergeDictJob.java
copy core-metadata/src/main/java/org/apache/kylin/metadata/model/ISourceAware.java => engine-mr/src/main/java/org/apache/kylin/engine/mr/streaming/MergeDictMapper.java (58%)
create mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/streaming/MergeDictReducer.java
copy core-metadata/src/main/java/org/apache/kylin/metadata/model/ISourceAware.java => engine-mr/src/main/java/org/apache/kylin/engine/mr/streaming/RowRecord.java (63%)
create mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/streaming/RowRecordReader.java
create mode 100644 engine-mr/src/main/java/org/apache/kylin/engine/mr/streaming/SaveDictStep.java
create mode 100644 examples/test_case_data/localmeta/cube/test_streaming_v2_cube.json
create mode 100644 examples/test_case_data/localmeta/cube/test_streaming_v2_user_info_cube.json
create mode 100644 examples/test_case_data/localmeta/cube_desc/test_streaming_v2_cube.json
create mode 100644 examples/test_case_data/localmeta/cube_desc/test_streaming_v2_user_info_cube.json
create mode 100644 examples/test_case_data/localmeta/data/DEFAULT.STREAMING_V2_USER_INFO_TABLE.csv
create mode 100644 examples/test_case_data/localmeta/model_desc/test_streaming_v2_model.json
create mode 100644 examples/test_case_data/localmeta/model_desc/test_streaming_v2_user_info_model.json
create mode 100644 examples/test_case_data/localmeta/streaming_v2/DEFAULT.STREAMING_V2_USER_INFO_TABLE.json
create mode 100644 examples/test_case_data/localmeta/table/DEFAULT.STREAMING_V2_TABLE.json
create mode 100644 examples/test_case_data/localmeta/table/DEFAULT.STREAMING_V2_USER_INFO_TABLE.json
create mode 100644 kylin-it/src/test/java/org/apache/kylin/realtime/BuildCubeWithStreamV2.java
create mode 100644 kylin-it/src/test/resources/query/sql_streaming_v2/compare_result/query01.sql
create mode 100644 kylin-it/src/test/resources/query/sql_streaming_v2/compare_result/query02.sql
create mode 100644 kylin-it/src/test/resources/query/sql_streaming_v2/compare_result/query03.sql
create mode 100644 kylin-it/src/test/resources/query/sql_streaming_v2/compare_result/query04.sql
create mode 100644 kylin-it/src/test/resources/query/sql_streaming_v2/compare_result/query05.sql
create mode 100644 kylin-it/src/test/resources/query/sql_streaming_v2/compare_result/query06.sql
create mode 100644 kylin-it/src/test/resources/query/sql_streaming_v2/compare_result/query07.sql
create mode 100644 kylin-it/src/test/resources/query/sql_streaming_v2/compare_result/query08.sql
create mode 100644 kylin-it/src/test/resources/query/sql_streaming_v2/count/query01.sql
create mode 100644 kylin-it/src/test/resources/query/sql_streaming_v2/not_compare_result/query01.sql
create mode 100644 kylin-it/src/test/resources/query/sql_streaming_v2/not_compare_result/query02.sql
create mode 100644 kylin-it/src/test/resources/streaming_v2_user_info_messages.txt
create mode 100644 server-base/src/main/java/org/apache/kylin/rest/controller/StreamingCoordinatorController.java
create mode 100644 server-base/src/main/java/org/apache/kylin/rest/controller/StreamingV2Controller.java
create mode 100644 server-base/src/main/java/org/apache/kylin/rest/request/StreamingRequestV2.java
create mode 100644 server-base/src/main/java/org/apache/kylin/rest/service/StreamingCoordinatorService.java
create mode 100644 server-base/src/main/java/org/apache/kylin/rest/service/StreamingV2Service.java
copy {engine-mr => storage-stream}/pom.xml (54%)
create mode 100644 storage-stream/src/main/java/org/apache/kylin/storage/stream/StreamStorage.java
create mode 100644 storage-stream/src/main/java/org/apache/kylin/storage/stream/StreamStorageQuery.java
create mode 100644 storage-stream/src/main/java/org/apache/kylin/storage/stream/rpc/HttpStreamDataSearchClient.java
create mode 100644 storage-stream/src/main/java/org/apache/kylin/storage/stream/rpc/IStreamDataSearchClient.java
create mode 100644 storage-stream/src/main/java/org/apache/kylin/storage/stream/rpc/MockedStreamDataSearchClient.java
copy {engine-mr => stream-coordinator}/pom.xml (55%)
create mode 100644 stream-coordinator/src/main/java/org/apache/kylin/stream/coordinator/Coordinator.java
create mode 100644 stream-coordinator/src/main/java/org/apache/kylin/stream/coordinator/HBaseStreamMetadataStore.java
create mode 100644 stream-coordinator/src/main/java/org/apache/kylin/stream/coordinator/MockStreamMetadataStore.java
create mode 100644 stream-coordinator/src/main/java/org/apache/kylin/stream/coordinator/StreamMetadataStore.java
create mode 100644 stream-coordinator/src/main/java/org/apache/kylin/stream/coordinator/StreamMetadataStoreFactory.java
create mode 100644 stream-coordinator/src/main/java/org/apache/kylin/stream/coordinator/StreamingCubeInfo.java
copy core-metadata/src/main/java/org/apache/kylin/metadata/model/ISourceAware.java => stream-coordinator/src/main/java/org/apache/kylin/stream/coordinator/StreamingUtils.java (60%)
create mode 100644 stream-coordinator/src/main/java/org/apache/kylin/stream/coordinator/ZookeeperStreamMetadataStore.java
create mode 100644 stream-coordinator/src/main/java/org/apache/kylin/stream/coordinator/assign/Assigner.java
create mode 100644 stream-coordinator/src/main/java/org/apache/kylin/stream/coordinator/assign/AssignmentUtil.java
create mode 100644 stream-coordinator/src/main/java/org/apache/kylin/stream/coordinator/assign/AssignmentsCache.java
create mode 100644 stream-coordinator/src/main/java/org/apache/kylin/stream/coordinator/assign/CubePartitionRoundRobinAssigner.java
create mode 100644 stream-coordinator/src/main/java/org/apache/kylin/stream/coordinator/assign/DefaultAssigner.java
create mode 100644 stream-coordinator/src/main/java/org/apache/kylin/stream/coordinator/client/CoordinatorClient.java
create mode 100644 stream-coordinator/src/main/java/org/apache/kylin/stream/coordinator/client/CoordinatorClientFactory.java
create mode 100644 stream-coordinator/src/main/java/org/apache/kylin/stream/coordinator/client/CoordinatorResponse.java
create mode 100644 stream-coordinator/src/main/java/org/apache/kylin/stream/coordinator/client/HttpCoordinatorClient.java
create mode 100644 stream-coordinator/src/main/java/org/apache/kylin/stream/coordinator/exception/ClusterStateException.java
copy core-metadata/src/main/java/org/apache/kylin/metadata/model/ISourceAware.java => stream-coordinator/src/main/java/org/apache/kylin/stream/coordinator/exception/CoordinateException.java (61%)
copy core-metadata/src/main/java/org/apache/kylin/metadata/model/ISourceAware.java => stream-coordinator/src/main/java/org/apache/kylin/stream/coordinator/exception/NotLeadCoordinatorException.java (60%)
copy core-metadata/src/main/java/org/apache/kylin/metadata/model/IStorageAware.java => stream-coordinator/src/main/java/org/apache/kylin/stream/coordinator/exception/StoreException.java (66%)
create mode 100644 stream-coordinator/src/test/java/org/apache/kylin/stream/coordinator/CoordinatorTest.java
create mode 100644 stream-coordinator/src/test/java/org/apache/kylin/stream/coordinator/assign/CubePartitionRoundRobinAssignerTest.java
create mode 100644 stream-coordinator/src/test/java/org/apache/kylin/stream/coordinator/assign/DefaultAssignerTest.java
copy {engine-mr => stream-core}/pom.xml (62%)
create mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/client/HttpReceiverAdminClient.java
create mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/client/ReceiverAdminClient.java
copy core-metadata/src/main/java/org/apache/kylin/metadata/model/IStorageAware.java => stream-core/src/main/java/org/apache/kylin/stream/core/consumer/ConsumerStartMode.java (75%)
create mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/consumer/ConsumerStartProtocol.java
create mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/consumer/EndPositionStopCondition.java
copy core-metadata/src/main/java/org/apache/kylin/metadata/model/IStorageAware.java => stream-core/src/main/java/org/apache/kylin/stream/core/consumer/IConsumerProvider.java (76%)
copy core-metadata/src/main/java/org/apache/kylin/metadata/model/IStorageAware.java => stream-core/src/main/java/org/apache/kylin/stream/core/consumer/IStopConsumptionCondition.java (65%)
copy webapp/app/js/app.js => stream-core/src/main/java/org/apache/kylin/stream/core/consumer/IStreamingConnector.java (55%)
mode change 100644 => 100755
copy webapp/app/js/app.js => stream-core/src/main/java/org/apache/kylin/stream/core/consumer/NeverStopConsumptionCondition.java (59%)
create mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/consumer/StreamingConsumerChannel.java
copy core-metadata/src/main/java/org/apache/kylin/metadata/model/IStorageAware.java => stream-core/src/main/java/org/apache/kylin/stream/core/exception/IllegalStorageException.java (64%)
copy core-metadata/src/main/java/org/apache/kylin/metadata/model/IStorageAware.java => stream-core/src/main/java/org/apache/kylin/stream/core/exception/StreamingException.java (65%)
create mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/metrics/StreamingMetrics.java
create mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/model/AssignRequest.java
copy core-metadata/src/main/java/org/apache/kylin/metadata/model/ISourceAware.java => stream-core/src/main/java/org/apache/kylin/stream/core/model/ConsumerStatsResponse.java (61%)
create mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/model/CubeAssignment.java
create mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/model/DataRequest.java
copy core-metadata/src/main/java/org/apache/kylin/metadata/model/ISourceAware.java => stream-core/src/main/java/org/apache/kylin/stream/core/model/DataResponse.java (63%)
copy core-metadata/src/main/java/org/apache/kylin/metadata/model/IStorageAware.java => stream-core/src/main/java/org/apache/kylin/stream/core/model/HealthCheckInfo.java (72%)
create mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/model/Node.java
copy core-metadata/src/main/java/org/apache/kylin/metadata/model/IStorageAware.java => stream-core/src/main/java/org/apache/kylin/stream/core/model/PauseConsumersRequest.java (68%)
create mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/model/RemoteStoreCompleteRequest.java
create mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/model/ReplicaSet.java
copy core-metadata/src/main/java/org/apache/kylin/metadata/model/ISourceAware.java => stream-core/src/main/java/org/apache/kylin/stream/core/model/ReplicaSetLeaderChangeRequest.java (55%)
copy core-metadata/src/main/java/org/apache/kylin/metadata/model/ISourceAware.java => stream-core/src/main/java/org/apache/kylin/stream/core/model/ResumeConsumerRequest.java (55%)
create mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/model/SegmentBuildState.java
copy core-metadata/src/main/java/org/apache/kylin/metadata/model/ISourceAware.java => stream-core/src/main/java/org/apache/kylin/stream/core/model/StartConsumersRequest.java (52%)
copy core-metadata/src/main/java/org/apache/kylin/metadata/model/ISourceAware.java => stream-core/src/main/java/org/apache/kylin/stream/core/model/StopConsumersRequest.java (56%)
copy core-metadata/src/main/java/org/apache/kylin/metadata/model/IStorageAware.java => stream-core/src/main/java/org/apache/kylin/stream/core/model/StreamingCubeConsumeState.java (76%)
create mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/model/StreamingMessage.java
copy core-metadata/src/main/java/org/apache/kylin/metadata/model/IStorageAware.java => stream-core/src/main/java/org/apache/kylin/stream/core/model/UnAssignRequest.java (69%)
create mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/model/stats/ClusterState.java
create mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/model/stats/ConsumerStats.java
create mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/model/stats/CubeRealTimeState.java
create mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/model/stats/LongLatencyInfo.java
create mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/model/stats/PartitionConsumeStats.java
create mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/model/stats/ReceiverCubeRealTimeState.java
create mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/model/stats/ReceiverCubeStats.java
create mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/model/stats/ReceiverState.java
create mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/model/stats/ReceiverStats.java
create mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/model/stats/ReplicaSetState.java
create mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/model/stats/SegmentStats.java
create mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/model/stats/SegmentStoreStats.java
create mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/query/HavingFilterChecker.java
copy core-metadata/src/main/java/org/apache/kylin/metadata/model/IStorageAware.java => stream-core/src/main/java/org/apache/kylin/stream/core/query/IStreamingGTSearcher.java (76%)
create mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/query/IStreamingSearchResult.java
create mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/query/MultiThreadsResultCollector.java
create mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/query/RecordsAggregator.java
create mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/query/ResponseResultSchema.java
create mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/query/ResultCollector.java
create mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/query/SingleThreadResultCollector.java
create mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/query/StreamingBuiltInFunctionTransformer.java
create mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/query/StreamingCubeDataSearcher.java
create mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/query/StreamingDataQueryPlanner.java
create mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/query/StreamingQueryProfile.java
create mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/query/StreamingSearchContext.java
create mode 100755 stream-core/src/main/java/org/apache/kylin/stream/core/query/StreamingTupleConverter.java
create mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/query/StreamingTupleIterator.java
copy core-metadata/src/main/java/org/apache/kylin/metadata/model/ISourceAware.java => stream-core/src/main/java/org/apache/kylin/stream/core/source/ISourcePosition.java (53%)
copy core-metadata/src/main/java/org/apache/kylin/metadata/model/ISourceAware.java => stream-core/src/main/java/org/apache/kylin/stream/core/source/ISourcePositionHandler.java (57%)
copy core-metadata/src/main/java/org/apache/kylin/metadata/model/IStorageAware.java => stream-core/src/main/java/org/apache/kylin/stream/core/source/IStreamingMessageParser.java (76%)
create mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/source/IStreamingSource.java
copy core-metadata/src/main/java/org/apache/kylin/metadata/model/IStorageAware.java => stream-core/src/main/java/org/apache/kylin/stream/core/source/MessageFormatException.java (65%)
create mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/source/MessageParserInfo.java
create mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/source/Partition.java
create mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/source/StreamingSourceConfig.java
create mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/source/StreamingSourceConfigManager.java
create mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/source/StreamingSourceFactory.java
copy core-metadata/src/main/java/org/apache/kylin/metadata/model/ISourceAware.java => stream-core/src/main/java/org/apache/kylin/stream/core/source/StreamingTableSourceInfo.java (50%)
create mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/storage/CheckPoint.java
create mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/storage/CheckPointStore.java
create mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/storage/IStreamingSegmentStore.java
create mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/storage/Record.java
create mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/storage/StreamingCubeSegment.java
create mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/storage/StreamingSegmentManager.java
copy core-metadata/src/main/java/org/apache/kylin/metadata/model/IStorageAware.java => stream-core/src/main/java/org/apache/kylin/stream/core/storage/columnar/ColumnDataReader.java (76%)
copy core-metadata/src/main/java/org/apache/kylin/metadata/model/IStorageAware.java => stream-core/src/main/java/org/apache/kylin/stream/core/storage/columnar/ColumnDataWriter.java (76%)
create mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/storage/columnar/ColumnarMemoryStorePersister.java
copy core-metadata/src/main/java/org/apache/kylin/metadata/model/ISourceAware.java => stream-core/src/main/java/org/apache/kylin/stream/core/storage/columnar/ColumnarMetricsEncoding.java (59%)
create mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/storage/columnar/ColumnarMetricsEncodingFactory.java
create mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/storage/columnar/ColumnarRecordCodec.java
create mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/storage/columnar/ColumnarSegmentStore.java
create mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/storage/columnar/ColumnarSegmentStoreFilesSearcher.java
create mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/storage/columnar/ColumnarStoreCache.java
create mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/storage/columnar/ColumnarStoreCacheStats.java
create mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/storage/columnar/ColumnarStoreDimDesc.java
create mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/storage/columnar/ColumnarStoreMetricsDesc.java
create mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/storage/columnar/DataSegmentFragment.java
create mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/storage/columnar/FSInputGeneralColumnDataReader.java
create mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/storage/columnar/FragmentCuboidReader.java
create mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/storage/columnar/FragmentData.java
create mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/storage/columnar/FragmentFileSearcher.java
create mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/storage/columnar/FragmentFilesMerger.java
create mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/storage/columnar/FragmentId.java
create mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/storage/columnar/FragmentSearchResult.java
create mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/storage/columnar/FragmentsMergeResult.java
create mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/storage/columnar/GeneralColumnDataReader.java
create mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/storage/columnar/GeneralColumnDataWriter.java
create mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/storage/columnar/ParsedStreamingCubeInfo.java
create mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/storage/columnar/RawRecord.java
create mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/storage/columnar/SegmentMemoryStore.java
create mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/storage/columnar/StringArrayComparator.java
create mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/storage/columnar/TimeDerivedColumnEncoding.java
copy core-metadata/src/main/java/org/apache/kylin/metadata/model/IStorageAware.java => stream-core/src/main/java/org/apache/kylin/stream/core/storage/columnar/compress/Compression.java (76%)
create mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/storage/columnar/compress/FSInputLZ4CompressedColumnReader.java
create mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/storage/columnar/compress/FSInputNoCompressedColumnReader.java
create mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/storage/columnar/compress/FSInputRLECompressedColumnReader.java
create mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/storage/columnar/compress/LZ4CompressedColumnReader.java
create mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/storage/columnar/compress/LZ4CompressedColumnWriter.java
create mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/storage/columnar/compress/LZ4CompressorTest.java
create mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/storage/columnar/compress/NoCompressedColumnReader.java
copy core-metadata/src/main/java/org/apache/kylin/metadata/model/ISourceAware.java => stream-core/src/main/java/org/apache/kylin/stream/core/storage/columnar/compress/NoCompressedColumnWriter.java (58%)
create mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/storage/columnar/compress/RunLengthCompressedColumnReader.java
create mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/storage/columnar/compress/RunLengthCompressedColumnWriter.java
create mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/storage/columnar/invertindex/ColInvertIndexSearcher.java
copy core-metadata/src/main/java/org/apache/kylin/metadata/model/ISourceAware.java => stream-core/src/main/java/org/apache/kylin/stream/core/storage/columnar/invertindex/ColInvertIndexWriter.java (53%)
create mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/storage/columnar/invertindex/FixLenColInvertIndexWriter.java
copy core-metadata/src/main/java/org/apache/kylin/metadata/model/ISourceAware.java => stream-core/src/main/java/org/apache/kylin/stream/core/storage/columnar/invertindex/FixLenIIColumnDescriptor.java (64%)
copy core-metadata/src/main/java/org/apache/kylin/metadata/model/IStorageAware.java => stream-core/src/main/java/org/apache/kylin/stream/core/storage/columnar/invertindex/IIColumnDescriptor.java (69%)
copy core-metadata/src/main/java/org/apache/kylin/metadata/model/IStorageAware.java => stream-core/src/main/java/org/apache/kylin/stream/core/storage/columnar/invertindex/IndexSearchResult.java (74%)
create mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/storage/columnar/invertindex/InvertIndexSearcher.java
create mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/storage/columnar/invertindex/SeqColInvertIndexWriter.java
copy core-metadata/src/main/java/org/apache/kylin/metadata/model/ISourceAware.java => stream-core/src/main/java/org/apache/kylin/stream/core/storage/columnar/invertindex/SeqIIColumnDescriptor.java (63%)
create mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/storage/columnar/protocol/CuboidMetaInfo.java
create mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/storage/columnar/protocol/DimDictionaryMetaInfo.java
create mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/storage/columnar/protocol/DimensionMetaInfo.java
create mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/storage/columnar/protocol/Footer.java
create mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/storage/columnar/protocol/FragmentMetaInfo.java
create mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/storage/columnar/protocol/MetricMetaInfo.java
create mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/storage/rocksdb/RocksDBSegmentStore.java
create mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/util/CompareFilterTimeRangeChecker.java
copy core-metadata/src/main/java/org/apache/kylin/metadata/model/IStorageAware.java => stream-core/src/main/java/org/apache/kylin/stream/core/util/Constants.java (76%)
create mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/util/HDFSUtil.java
copy core-metadata/src/main/java/org/apache/kylin/metadata/model/ISourceAware.java => stream-core/src/main/java/org/apache/kylin/stream/core/util/NamedThreadFactory.java (50%)
create mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/util/NodeUtil.java
create mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/util/RecordsSerializer.java
create mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/util/RestService.java
copy core-metadata/src/main/java/org/apache/kylin/metadata/model/IStorageAware.java => stream-core/src/main/java/org/apache/kylin/stream/core/util/RetryCallable.java (68%)
create mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/util/RetryCaller.java
create mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/util/StreamFilterUtil.java
create mode 100644 stream-core/src/main/java/org/apache/kylin/stream/core/util/TimeDerivedColumnType.java
create mode 100644 stream-core/src/test/java/org/apache/kylin/stream/core/storage/CheckPointStoreTest.java
create mode 100644 stream-core/src/test/java/org/apache/kylin/stream/core/storage/MockPositionHandler.java
create mode 100644 stream-core/src/test/java/org/apache/kylin/stream/core/storage/RecordsSerDeTest.java
create mode 100644 stream-core/src/test/java/org/apache/kylin/stream/core/storage/StreamingSegmentManagerTest.java
create mode 100644 stream-core/src/test/java/org/apache/kylin/stream/core/storage/TestHelper.java
create mode 100644 stream-core/src/test/java/org/apache/kylin/stream/core/storage/columnar/ColumnarSegmentStoreTest.java
create mode 100644 stream-core/src/test/java/org/apache/kylin/stream/core/storage/columnar/FragmentCuboidReaderTest.java
create mode 100644 stream-core/src/test/java/org/apache/kylin/stream/core/storage/columnar/FragmentFileSearcherTest.java
create mode 100644 stream-core/src/test/java/org/apache/kylin/stream/core/storage/columnar/FragmentFilesMergerTest.java
create mode 100644 stream-core/src/test/java/org/apache/kylin/stream/core/storage/columnar/GeneralColumnDataTest.java
create mode 100644 stream-core/src/test/java/org/apache/kylin/stream/core/storage/columnar/SegmentMemoryStoreTest.java
create mode 100644 stream-core/src/test/java/org/apache/kylin/stream/core/storage/columnar/StreamingDataSimulator.java
create mode 100644 stream-core/src/test/java/org/apache/kylin/stream/core/storage/columnar/compress/LZ4CompressColumnTest.java
create mode 100644 stream-core/src/test/java/org/apache/kylin/stream/core/storage/columnar/compress/NoCompressColumnTest.java
create mode 100644 stream-core/src/test/java/org/apache/kylin/stream/core/storage/columnar/compress/RunLengthCompressColumnTest.java
create mode 100644 stream-core/src/test/java/org/apache/kylin/stream/core/storage/columnar/invertindex/ColInvertIndexWriterWriterTest.java
copy webapp/app/js/app.js => stream-core/src/test/java/org/apache/kylin/stream/core/storage/columnar/invertindex/SearchableColInvertIndexWriterTest.java (51%)
create mode 100644 stream-core/src/test/java/org/apache/kylin/stream/core/storage/columnar/invertindex/SimpleColInvertIndexWriterTest.java
create mode 100644 stream-core/src/test/java/org/apache/kylin/stream/core/storage/columnar/performance/FragmentCuboidReaderPerfTest.java
create mode 100644 stream-core/src/test/java/org/apache/kylin/stream/core/storage/columnar/performance/PerfDataPrepare.java
create mode 100644 stream-core/src/test/java/org/apache/kylin/stream/core/storage/columnar/performance/PerformanceTest.java
create mode 100644 stream-core/src/test/java/org/apache/kylin/stream/core/storage/columnar/performance/StreamingCubeDataSearcherPerfTest.java
create mode 100644 stream-core/src/test/java/org/apache/kylin/stream/core/util/DataGenerator.java
create mode 100644 stream-core/src/test/java/org/apache/kylin/stream/core/util/TimeDerivedColumnTypeTest.java
create mode 100644 stream-receiver/pom.xml
create mode 100644 stream-receiver/src/main/java/org/apache/kylin/stream/server/ReplicaSetLeaderSelector.java
copy core-metadata/src/main/java/org/apache/kylin/metadata/model/ISourceAware.java => stream-receiver/src/main/java/org/apache/kylin/stream/server/ServerContext.java (61%)
create mode 100644 stream-receiver/src/main/java/org/apache/kylin/stream/server/StreamingReceiver.java
create mode 100644 stream-receiver/src/main/java/org/apache/kylin/stream/server/StreamingServer.java
create mode 100644 stream-receiver/src/main/java/org/apache/kylin/stream/server/rest/controller/AdminController.java
create mode 100644 stream-receiver/src/main/java/org/apache/kylin/stream/server/rest/controller/BasicController.java
create mode 100644 stream-receiver/src/main/java/org/apache/kylin/stream/server/rest/controller/DataController.java
create mode 100644 stream-receiver/src/main/java/org/apache/kylin/stream/server/rest/controller/QueryController.java
create mode 100644 stream-receiver/src/main/java/org/apache/kylin/stream/server/rest/controller/StatsController.java
create mode 100644 stream-receiver/src/main/java/org/apache/kylin/stream/server/rest/controller/SystemController.java
copy core-metadata/src/main/java/org/apache/kylin/metadata/model/ISourceAware.java => stream-receiver/src/main/java/org/apache/kylin/stream/server/rest/exception/BadRequestException.java (50%)
copy core-metadata/src/main/java/org/apache/kylin/metadata/model/ISourceAware.java => stream-receiver/src/main/java/org/apache/kylin/stream/server/rest/exception/ForbiddenException.java (61%)
create mode 100644 stream-receiver/src/main/java/org/apache/kylin/stream/server/rest/exception/InternalErrorException.java
copy core-metadata/src/main/java/org/apache/kylin/metadata/model/IStorageAware.java => stream-receiver/src/main/java/org/apache/kylin/stream/server/rest/exception/NotFoundException.java (66%)
copy core-metadata/src/main/java/org/apache/kylin/metadata/model/IStorageAware.java => stream-receiver/src/main/java/org/apache/kylin/stream/server/rest/model/ErrorResponse.java (69%)
create mode 100644 stream-receiver/src/main/java/org/apache/kylin/stream/server/rest/model/PrepareSqlRequest.java
create mode 100644 stream-receiver/src/main/java/org/apache/kylin/stream/server/rest/model/SQLRequest.java
create mode 100644 stream-receiver/src/main/java/org/apache/kylin/stream/server/rest/model/SQLResponse.java
create mode 100644 stream-receiver/src/main/java/org/apache/kylin/stream/server/rest/security/StreamTableInterceptor.java
create mode 100644 stream-receiver/src/main/java/org/apache/kylin/stream/server/rest/service/BasicService.java
create mode 100644 stream-receiver/src/main/java/org/apache/kylin/stream/server/rest/service/QueryService.java
create mode 100644 stream-receiver/src/main/java/org/apache/kylin/stream/server/rest/util/QueryUtil.java
create mode 100644 stream-receiver/src/main/java/org/apache/kylin/stream/server/retention/RetentionPolicyInfo.java
copy core-metadata/src/main/java/org/apache/kylin/metadata/model/ISourceAware.java => stream-receiver/src/main/java/org/apache/kylin/stream/server/storage/LocalStreamStorage.java (57%)
create mode 100644 stream-receiver/src/main/java/org/apache/kylin/stream/server/storage/LocalStreamStorageQuery.java
create mode 100644 stream-receiver/src/main/resources/applicationContext.xml
create mode 100644 stream-receiver/src/main/resources/ehcache-test.xml
create mode 100644 stream-receiver/src/main/resources/ehcache.xml
create mode 100644 stream-receiver/src/main/resources/stream-receiver-log4j.properties
create mode 100644 stream-receiver/src/main/webapp/WEB-INF/kylin-servlet.xml
create mode 100644 stream-receiver/src/main/webapp/index.html
create mode 100644 stream-source-kafka/pom.xml
create mode 100644 stream-source-kafka/src/main/java/org/apache/kylin/stream/source/kafka/BootstrapServerConfig.java
create mode 100644 stream-source-kafka/src/main/java/org/apache/kylin/stream/source/kafka/KafkaBatchSourceAdaptor.java
copy core-metadata/src/main/java/org/apache/kylin/metadata/model/ISourceAware.java => stream-source-kafka/src/main/java/org/apache/kylin/stream/source/kafka/KafkaConsumerStartInfo.java (56%)
create mode 100644 stream-source-kafka/src/main/java/org/apache/kylin/stream/source/kafka/KafkaPosition.java
create mode 100644 stream-source-kafka/src/main/java/org/apache/kylin/stream/source/kafka/KafkaPositionHandler.java
create mode 100644 stream-source-kafka/src/main/java/org/apache/kylin/stream/source/kafka/KafkaSource.java
create mode 100644 stream-source-kafka/src/main/java/org/apache/kylin/stream/source/kafka/KafkaTopicAssignment.java
create mode 100644 stream-source-kafka/src/main/java/org/apache/kylin/stream/source/kafka/TimedJsonStreamParser.java
create mode 100644 stream-source-kafka/src/main/java/org/apache/kylin/stream/source/kafka/consumer/KafkaConnector.java
create mode 100644 stream-source-kafka/src/test/java/org/apache/kylin/stream/coordinator/assign/KafkaSourcePositionHandlerTest.java
create mode 100644 webapp/app/js/controllers/adminStreaming.js
create mode 100644 webapp/app/js/controllers/streamingBalanceAssignGroup.js
create mode 100644 webapp/app/js/utils/liquidFillGauge.js
copy webapp/app/partials/admin/{admin.html => config.html} (100%)
create mode 100644 webapp/app/partials/admin/streaming.html
create mode 100644 webapp/app/partials/admin/streamingReceiver.html
create mode 100644 webapp/app/partials/streaming/balanceReplicaSet.html
create mode 100644 webapp/app/partials/streaming/cubeAssignment.html
create mode 100644 webapp/app/partials/tables/loadStreamingSourceConfig.html
create mode 100644 webapp/app/partials/tables/loadStreamingTableConfig.html