You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by xi...@apache.org on 2020/03/18 21:06:08 UTC
[incubator-pinot] branch update-readme updated (f356f7e -> 858fe64)
This is an automated email from the ASF dual-hosted git repository.
xiangfu pushed a change to branch update-readme
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git.
from f356f7e Updating readme
add 76f9c53 Support schema evolution for consuming segments (#4954)
add a5fb1bd docker - generator script for synthetic time series in pinot (#5134)
add 3e451e7 Refactor the data source to include all information needed for query execution (#5115)
add 330f37f Fix the NPE when no segment exists in RealtimeSegmentSelector (#5138)
add 5c7ece2 Add test for RealtimeSegmentSelector for no segment scenario (#5139)
add d8fabec [TE] database - add missing schema fields and index to application_index (#5124)
add 6d8c60a [TE] Minor corrections to the default yaml template (#5140)
add 46698d8 Update links in README to latest docs (#5141)
add 6daeb59 Fix the default value provider classes (#5137)
add f7f9f4a Fix hybrid quickstart (#5143)
add 6b6fa56a Update license and notice (#5145)
add 888700d [TE] Remove validation to make recipients backwards compatible (#5149)
add 82cfbdd Fix a formatting bug in AggregationFunctionUtils (#5148)
add d15a91a update pinot assembly scripts (#5146)
add d989427 Fix the SQL group-by for empty data table (#5151)
add 494874d [TE] Detection creation endpoints should trigger Replay & Tuning (#5142)
add be37b56 Refactor value based segment pruner to work on DataSource instead of ColumnMetadata (#5144)
add 9933870 Move SegmentMetadata from pinot-common to pinot-core (#5156)
add 4be032a [TE] frontend - harleyjj/alert-details - stop showing bounds for minutely granularity on Alert Overview (#5157)
add 1376331 [TE] Setup a test Github workflow for ThirdEye (#5158)
add c81a656 Add Azure Data Lake Gen2 connector for PinotFS (#5116)
add bbaa1d9 [TE] anomaly detection model downloader (#5155)
add 8eb5933 Updated Readme with Pinot clients and replace gitter badge with slack (#5164)
add 858fe64 Merge branch 'master' into update-readme
No new revisions were added by this update.
Summary of changes:
.github/workflows/maven.yml | 43 ++
LICENSE-binary | 131 +++++-
NOTICE | 2 +-
NOTICE-binary | 448 ++++++++++++--------
README.md | 22 +-
docker/images/pinot/Dockerfile | 1 +
docker/images/pinot/bin/generator.sh | 77 ++++
docs/index.rst | 2 +
licenses-binary/LICENSE-fusesource.txt | 27 ++
licenses-binary/LICENSE-py4j.txt | 26 ++
licenses-binary/LICENSE-threeten.txt | 29 ++
.../pinot/broker/routing/RoutingManager.java | 15 +-
.../segmentselector/RealtimeSegmentSelector.java | 3 +
.../segmentselector/SegmentSelectorTest.java | 12 +-
.../pinot/common/config/ColumnPartitionConfig.java | 7 +-
.../common/config/SegmentPartitionConfig.java | 13 -
.../apache/pinot/common/utils/CommonConstants.java | 4 +
.../resources/LLCSegmentCompletionHandlers.java | 2 +-
.../PinotSegmentUploadDownloadRestletResource.java | 2 +-
.../controller/api/upload/SegmentValidator.java | 4 +-
.../pinot/controller/api/upload/ZKOperator.java | 2 +-
.../helix/core/PinotHelixResourceManager.java | 2 +-
.../realtime/PinotLLCRealtimeSegmentManager.java | 8 +-
.../segment/CommittingSegmentDescriptor.java | 2 +-
.../helix/core/util/ZKMetadataUtils.java | 6 +-
.../api/PinotSegmentRestletResourceTest.java | 4 +-
.../controller/api/upload/ZKOperatorTest.java | 2 +-
.../PinotLLCRealtimeSegmentManagerTest.java | 2 +-
.../helix/core/retention/RetentionManagerTest.java | 2 +-
.../controller/utils/SegmentMetadataMockUtils.java | 6 +-
.../validation/ValidationManagerTest.java | 2 +-
.../org/apache/pinot/core/common/DataFetcher.java | 14 +-
.../org/apache/pinot/core/common/DataSource.java | 25 +-
.../pinot/core/common/DataSourceMetadata.java | 69 ++-
.../core/common/datatable/DataTableBuilder.java | 113 +----
.../core/common/datatable/DataTableUtils.java | 123 +++++-
.../core/data/manager/InstanceDataManager.java | 2 +-
.../realtime/LLRealtimeSegmentDataManager.java | 36 +-
.../data/readers/PinotSegmentColumnReader.java | 6 +-
.../data/readers/PinotSegmentRecordReader.java | 10 +-
.../pinot/core/indexsegment/IndexSegment.java | 7 +-
.../immutable/ImmutableSegmentImpl.java | 15 +-
.../immutable/ImmutableSegmentLoader.java | 7 +-
.../core/indexsegment/mutable/MutableSegment.java | 6 +-
.../indexsegment/mutable/MutableSegmentImpl.java | 129 ++++--
.../reader/impl/ConstantMVForwardIndex.java} | 34 +-
.../reader/impl/ConstantMVInvertedIndex.java} | 35 +-
.../reader/impl/ConstantSVSortedIndex.java} | 45 +-
.../io/reader/impl/v1/SortedIndexReaderImpl.java | 1 -
.../core/metadata/DefaultMetadataExtractor.java | 4 +-
.../pinot/core/metadata/MetadataExtractor.java | 2 +-
.../pinot/core/minion/BackfillDateTimeColumn.java | 4 +-
.../pinot/core/minion/RawIndexConverter.java | 10 +-
.../apache/pinot/core/minion/SegmentPurger.java | 2 +-
.../minion/rollup/MergeRollupSegmentConverter.java | 6 +-
.../core/operator/filter/FilterOperatorUtils.java | 20 +-
.../SortedInvertedIndexBasedFilterOperator.java | 1 -
.../query/MetadataBasedAggregationOperator.java | 2 +-
.../function/IdentifierTransformFunction.java | 8 +-
.../plan/MetadataBasedAggregationPlanNode.java | 2 +-
.../function/AggregationFunctionUtils.java | 13 +-
.../core/query/config/SegmentPrunerConfig.java | 53 +--
.../query/executor/ServerQueryExecutorV1Impl.java | 6 +-
.../core/query/pruner/AbstractSegmentPruner.java | 105 -----
.../query/pruner/ColumnValueSegmentPruner.java | 312 ++++++++------
.../core/query/pruner/DataSchemaSegmentPruner.java | 4 +-
.../core/query/pruner/PartitionSegmentPruner.java | 117 ------
.../core/query/pruner/SegmentPrunerProvider.java | 1 -
.../core/query/pruner/SegmentPrunerService.java | 4 +-
.../core/query/pruner/ValidSegmentPruner.java | 2 +-
.../core/query/reduce/BrokerReduceService.java | 6 +
.../core/query/reduce/GroupByDataTableReducer.java | 8 -
.../query/reduce/SelectionDataTableReducer.java | 29 +-
.../query/selection/SelectionOperatorUtils.java | 13 +-
.../converter/stats/RealtimeColumnStatistics.java | 99 ++---
.../stats/RealtimeNoDictionaryColStatistics.java | 41 +-
.../stats/RealtimeSegmentStatsContainer.java | 14 +-
.../core/realtime/impl/RealtimeSegmentConfig.java | 54 ++-
.../impl/dictionary/BaseMutableDictionary.java | 4 +-
.../column}/BaseVirtualColumnProvider.java | 13 +-
.../DefaultNullValueVirtualColumnProvider.java | 87 ++++
.../index/column/PhysicalColumnIndexContainer.java | 7 +-
.../converter/SegmentV1V2ToV3FormatConverter.java | 2 +-
.../index/data/source/ColumnDataSource.java | 201 ---------
.../index/datasource/BaseDataSource.java} | 109 ++---
.../index/datasource/ImmutableDataSource.java | 119 ++++++
.../index/datasource/MutableDataSource.java | 123 ++++++
.../segment/index/loader/SegmentPreProcessor.java | 6 +-
.../loader/bloomfilter/BloomFilterHandler.java | 6 +-
.../ColumnMinMaxValueGenerator.java | 8 +-
.../defaultcolumn/BaseDefaultColumnHandler.java | 6 +-
.../defaultcolumn/DefaultColumnHandlerFactory.java | 4 +-
.../defaultcolumn/V1DefaultColumnHandler.java | 2 +-
.../defaultcolumn/V3DefaultColumnHandler.java | 8 +-
.../loader/invertedindex/InvertedIndexHandler.java | 4 +-
.../loader/invertedindex/TextIndexHandler.java | 5 +-
.../index/{ => metadata}/ColumnMetadata.java | 29 +-
.../segment/index/metadata}/SegmentMetadata.java | 4 +-
.../index/{ => metadata}/SegmentMetadataImpl.java | 3 +-
.../readers/ConstantValueBytesDictionary.java | 82 ++++
.../readers/ConstantValueDoubleDictionary.java} | 53 ++-
.../readers/ConstantValueFloatDictionary.java} | 53 ++-
.../readers/ConstantValueIntDictionary.java} | 51 ++-
.../readers/ConstantValueLongDictionary.java} | 51 ++-
.../readers/ConstantValueStringDictionary.java | 81 ++++
.../segment/index/readers/DocIdDictionary.java | 73 ++++
.../core/segment/store/ColumnIndexDirectory.java | 3 +-
.../core/segment/store/FilePerIndexDirectory.java | 2 +-
.../pinot/core/segment/store/SegmentDirectory.java | 2 +-
.../segment/store/SegmentLocalFSDirectory.java | 2 +-
.../segment/store/SingleFileIndexDirectory.java | 2 +-
.../virtualcolumn/DocIdVirtualColumnProvider.java | 59 +--
.../SingleStringVirtualColumnProvider.java | 160 -------
.../virtualcolumn/VirtualColumnContext.java | 2 +-
.../virtualcolumn/VirtualColumnProvider.java | 2 +-
.../VirtualColumnProviderFactory.java | 19 +-
.../core/startree/v2/store/StarTreeDataSource.java | 98 +++++
.../startree/v2/store/StarTreeIndexContainer.java | 2 +-
.../startree/v2/store/StarTreeLoaderUtils.java | 33 +-
.../v2/store/StarTreeMetricDataSource.java | 136 ------
.../pinot/core/util/FixedIntArrayOffHeapIdMap.java | 4 +-
.../core/common/RealtimeNoDictionaryTest.java | 33 +-
.../core/common/datatable/DataTableUtilsTest.java | 86 ++++
.../data/manager/BaseTableDataManagerTest.java | 2 +-
.../MutableSegmentImplNullValueVectorTest.java | 6 +-
.../mutable/MutableSegmentImplTest.java | 12 +-
.../pinot/core/minion/SegmentPurgerTest.java | 10 +-
.../function/AggregationFunctionUtilsTest.java | 66 +++
.../query/pruner/ColumnValueSegmentPrunerTest.java | 127 ++++++
.../core/segment/index/ColumnMetadataTest.java | 2 +
.../segment/index/SegmentMetadataImplTest.java | 2 +
.../DefaultNullValueVirtualColumnProviderTest.java | 177 ++++++++
.../SegmentV1V2ToV3FormatConverterTest.java | 2 +-
.../SegmentGenerationWithBytesTypeTest.java | 20 +-
.../SegmentGenerationWithTimeColumnTest.java | 12 +-
.../index/creator/SegmentPartitionTest.java | 75 +---
.../core/segment/index/loader/LoaderTest.java | 19 +-
.../index/loader/SegmentPreProcessorTest.java | 4 +-
.../BaseDefaultColumnHandlerTest.java | 208 ++++++++++
.../store/ColumnIndexDirectoryTestHelper.java | 2 +-
.../segment/store/FilePerIndexDirectoryTest.java | 2 +-
.../segment/store/SegmentLocalFSDirectoryTest.java | 4 +-
.../store/SingleFileIndexDirectoryTest.java | 2 +-
.../org/apache/pinot/core/util/CrcUtilsTest.java | 2 +-
.../InterSegmentOrderBySingleValueQueriesTest.java | 68 ++-
.../query/pruner/ColumnValueSegmentPrunerTest.java | 100 -----
.../selection/SelectionOperatorServiceTest.java | 2 +-
.../pinot/segments/v1/creator/IntArraysTest.java | 6 +-
.../test/resources/conf/query-executor.properties | 6 +-
pinot-distribution/pinot-assembly.xml | 88 ++--
.../tests/BaseClusterIntegrationTestSet.java | 113 ++++-
...vertToRawIndexMinionClusterIntegrationTest.java | 4 +-
.../tests/HybridClusterIntegrationTest.java | 12 +
.../tests/LLCRealtimeClusterIntegrationTest.java | 7 +
.../MultiNodesOfflineClusterIntegrationTest.java | 21 +
.../src/test/resources/log4j2.xml | 2 +-
.../pinot/perf/BenchmarkOfflineIndexReader.java | 4 +-
.../apache/pinot/perf/BenchmarkQueryEngine.java | 4 +-
pinot-plugins/pinot-file-system/pinot-adls/pom.xml | 6 +
.../pinot/plugin/filesystem/AzureGen2PinotFS.java | 462 +++++++++++++++++++++
.../pinot/plugin/filesystem/AzurePinotFSUtil.java | 89 ++++
.../filesystem/test/AzurePinotFSUtilTest.java | 101 +++++
.../apache/pinot/plugin/filesystem/GcsPinotFS.java | 4 +-
pinot-plugins/pom.xml | 1 -
.../pinot/server/api/resources/TablesResource.java | 2 +-
.../helix/DefaultHelixStarterServerConfig.java | 8 +-
.../starter/helix/HelixInstanceDataManager.java | 29 +-
.../helix/HelixInstanceDataManagerConfig.java | 8 +
.../starter/helix/SegmentFetcherAndLoader.java | 4 +-
.../pinot/server/api/TablesResourceTest.java | 4 +-
.../java/org/apache/pinot/spi/data/FieldSpec.java | 4 +
.../org/apache/pinot/spi/filesystem/PinotFS.java | 16 +-
.../pinot/spi/filesystem/LocalPinotFSTest.java | 8 +
.../org/apache/pinot/tools/HybridQuickstart.java | 8 +-
.../org/apache/pinot/tools/SegmentDumpTool.java | 6 +-
.../apache/pinot/tools/StarTreeIndexViewer.java | 4 +-
.../anonymizer/ArrayBasedGlobalDictionaries.java | 3 +-
.../pinot/tools/anonymizer/GlobalDictionaries.java | 2 +-
.../anonymizer/MapBasedGlobalDictionaries.java | 2 +-
.../anonymizer/PinotDataAndQueryAnonymizer.java | 27 +-
.../pinot/tools/perf/PerfBenchmarkDriver.java | 6 +-
.../pinot/tools/perf/PerfBenchmarkRunner.java | 2 +-
.../realtime/provisioning/MemoryEstimator.java | 8 +-
.../apache/pinot/tools/scan/query/Aggregation.java | 2 +-
.../apache/pinot/tools/scan/query/Projection.java | 4 +-
.../tools/scan/query/SegmentQueryProcessor.java | 12 +-
.../apache/pinot/tools/scan/query/Selection.java | 2 +-
.../segment/converter/SegmentMergeCommand.java | 6 +-
.../pinot/tools/streams/AirlineDataStream.java | 12 +-
pom.xml | 45 +-
.../app/pods/components/alert-details/component.js | 4 +-
thirdeye/thirdeye-frontend/app/utils/yaml-tools.js | 15 +-
.../components/detection-yaml/component-test.js | 8 +-
.../components/subscription-yaml/component-test.js | 4 +-
.../anomaly/ThirdEyeAnomalyApplication.java | 9 +
.../thirdeye/common/ThirdEyeConfiguration.java | 11 +
.../dashboard/ThirdEyeDashboardApplication.java | 10 +
.../alert/StatefulDetectionAlertFilter.java | 3 +-
.../alert/scheme/DetectionEmailAlerter.java | 5 +-
.../validators/SubscriptionConfigValidator.java | 4 -
.../thirdeye/detection/yaml/YamlResource.java | 50 ++-
.../thirdeye/model/download/ModelDownloader.java | 37 +-
.../download/ModelDownloaderConfiguration.java | 65 +++
.../model/download/ModelDownloaderManager.java | 88 ++++
.../src/main/resources/schema/create-schema.sql | 8 +-
.../thirdeye/detection/yaml/YamlResourceTest.java | 2 +-
206 files changed, 4499 insertions(+), 2280 deletions(-)
create mode 100644 .github/workflows/maven.yml
create mode 100755 docker/images/pinot/bin/generator.sh
create mode 100644 licenses-binary/LICENSE-fusesource.txt
create mode 100644 licenses-binary/LICENSE-py4j.txt
create mode 100644 licenses-binary/LICENSE-threeten.txt
rename pinot-core/src/main/java/org/apache/pinot/core/{segment/virtualcolumn/DefaultNullValueSingleStringVirtualColumnProvider.java => io/reader/impl/ConstantMVForwardIndex.java} (53%)
copy pinot-core/src/main/java/org/apache/pinot/core/{segment/virtualcolumn/VirtualColumnProvider.java => io/reader/impl/ConstantMVInvertedIndex.java} (53%)
copy pinot-core/src/main/java/org/apache/pinot/core/{segment/virtualcolumn/IntSingleValueDataFileReader.java => io/reader/impl/ConstantSVSortedIndex.java} (57%)
delete mode 100644 pinot-core/src/main/java/org/apache/pinot/core/query/pruner/AbstractSegmentPruner.java
delete mode 100644 pinot-core/src/main/java/org/apache/pinot/core/query/pruner/PartitionSegmentPruner.java
rename pinot-core/src/main/java/org/apache/pinot/core/segment/{virtualcolumn => index/column}/BaseVirtualColumnProvider.java (71%)
create mode 100644 pinot-core/src/main/java/org/apache/pinot/core/segment/index/column/DefaultNullValueVirtualColumnProvider.java
delete mode 100644 pinot-core/src/main/java/org/apache/pinot/core/segment/index/data/source/ColumnDataSource.java
rename pinot-core/src/main/java/org/apache/pinot/core/{startree/v2/store/StarTreeDimensionDataSource.java => segment/index/datasource/BaseDataSource.java} (50%)
create mode 100644 pinot-core/src/main/java/org/apache/pinot/core/segment/index/datasource/ImmutableDataSource.java
create mode 100644 pinot-core/src/main/java/org/apache/pinot/core/segment/index/datasource/MutableDataSource.java
rename pinot-core/src/main/java/org/apache/pinot/core/segment/index/{ => metadata}/ColumnMetadata.java (90%)
rename {pinot-common/src/main/java/org/apache/pinot/common/segment => pinot-core/src/main/java/org/apache/pinot/core/segment/index/metadata}/SegmentMetadata.java (95%)
rename pinot-core/src/main/java/org/apache/pinot/core/segment/index/{ => metadata}/SegmentMetadataImpl.java (99%)
create mode 100644 pinot-core/src/main/java/org/apache/pinot/core/segment/index/readers/ConstantValueBytesDictionary.java
copy pinot-core/src/main/java/org/apache/pinot/core/segment/{virtualcolumn/IntSingleValueDataFileReader.java => index/readers/ConstantValueDoubleDictionary.java} (50%)
copy pinot-core/src/main/java/org/apache/pinot/core/segment/{virtualcolumn/IntSingleValueDataFileReader.java => index/readers/ConstantValueFloatDictionary.java} (50%)
copy pinot-core/src/main/java/org/apache/pinot/core/segment/{virtualcolumn/IntSingleValueDataFileReader.java => index/readers/ConstantValueIntDictionary.java} (50%)
rename pinot-core/src/main/java/org/apache/pinot/core/segment/{virtualcolumn/IntSingleValueDataFileReader.java => index/readers/ConstantValueLongDictionary.java} (50%)
create mode 100644 pinot-core/src/main/java/org/apache/pinot/core/segment/index/readers/ConstantValueStringDictionary.java
create mode 100644 pinot-core/src/main/java/org/apache/pinot/core/segment/index/readers/DocIdDictionary.java
delete mode 100644 pinot-core/src/main/java/org/apache/pinot/core/segment/virtualcolumn/SingleStringVirtualColumnProvider.java
create mode 100644 pinot-core/src/main/java/org/apache/pinot/core/startree/v2/store/StarTreeDataSource.java
delete mode 100644 pinot-core/src/main/java/org/apache/pinot/core/startree/v2/store/StarTreeMetricDataSource.java
create mode 100644 pinot-core/src/test/java/org/apache/pinot/core/common/datatable/DataTableUtilsTest.java
create mode 100644 pinot-core/src/test/java/org/apache/pinot/core/query/aggregation/function/AggregationFunctionUtilsTest.java
create mode 100644 pinot-core/src/test/java/org/apache/pinot/core/query/pruner/ColumnValueSegmentPrunerTest.java
create mode 100644 pinot-core/src/test/java/org/apache/pinot/core/segment/index/column/DefaultNullValueVirtualColumnProviderTest.java
create mode 100644 pinot-core/src/test/java/org/apache/pinot/core/segment/index/loader/defaultcolumn/BaseDefaultColumnHandlerTest.java
delete mode 100644 pinot-core/src/test/java/org/apache/pinot/query/pruner/ColumnValueSegmentPrunerTest.java
create mode 100644 pinot-plugins/pinot-file-system/pinot-adls/src/main/java/org/apache/pinot/plugin/filesystem/AzureGen2PinotFS.java
create mode 100644 pinot-plugins/pinot-file-system/pinot-adls/src/main/java/org/apache/pinot/plugin/filesystem/AzurePinotFSUtil.java
create mode 100644 pinot-plugins/pinot-file-system/pinot-adls/src/test/java/org/apache/pinot/plugin/filesystem/test/AzurePinotFSUtilTest.java
copy pinot-core/src/main/java/org/apache/pinot/core/metadata/MetadataExtractor.java => thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/model/download/ModelDownloader.java (51%)
create mode 100644 thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/model/download/ModelDownloaderConfiguration.java
create mode 100644 thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/model/download/ModelDownloaderManager.java
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org