You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by ja...@apache.org on 2020/12/28 21:48:12 UTC
[incubator-pinot] branch json-indexing updated (b8aab32 -> c217c9e)
This is an automated email from the ASF dual-hosted git repository.
jackie pushed a change to branch json-indexing
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git.
discard b8aab32 Add json index support
add bdeec8d Fixing the issue with result schema (#6353)
add 9644350 add redis module dependency (#6357)
add 946ff55 [TE] fix javassist version (#6356)
add fb42e72 [TE]frontend - Add breadcrumb component (#6350)
add a76e766 Broker time range pruning(#6189) (#6259)
add 5432099 [TE]frontend - Add a lightweight PubSub system (#6358)
add 8e41708 6355 don't lose authority portion of inputDirURI (#6359)
add 2ea0185 Enhance task schedule api for single type/table support (#6352)
add 79b12ed Fix empty data table for distinct query (#6363)
add 0522acc pinot-controller unit test suite. (#6326)
add 003442a [TE]frontend - Fix the display for "NaN" values in performance-stats (#6365)
add 39a24cf [TE]frontend - THIRDEYE-3772 - create a new 'composite-anomalies' component plus very simple unit integration test (#6369)
add 75b0792 Fix mvn profile name for github-actions. (#6370)
add 2a7d506 Improve and bug fix on json record extraction logic (#6372)
add 85cf696 Fixing the issue for realtime table creation with env variables in stream configs (#6373)
add 10dad7d Ingestion resource with APIs for ingestion via file/URI (#6354)
add b8bc74f compatibility test: create/delete table operations (#6360)
add 7580ae4 [TE]frontend - Integrate Performance Stats into Composite Anomalies (#6366)
add 33de6dc Introduce DimensionTableDataManager (#6346)
add 6d8b09e Update Helm readme with latest incubator repo (#6381)
add 1bb6c14 Add FST index which works on top of REGEXP_LIKE operator. (#6120)
add fed6750 Adding a version option to pinot admin to show all the component versions (#6380)
add e268a79 Adding S3PinotFS as one of the default PinotFS for Quickstart (#6379)
add c217c9e Add json index support
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (b8aab32)
\
N -- N -- N refs/heads/json-indexing (c217c9e)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
No new revisions were added by this update.
Summary of changes:
.github/workflows/scripts/.pinot_test.sh | 4 +-
.gitignore | 2 +
docker/images/pinot-superset/requirements-db.txt | 1 +
kubernetes/helm/README.md | 4 +-
.../pinot/broker/routing/RoutingManager.java | 4 +-
.../instanceselector/BaseInstanceSelector.java | 2 -
.../segmentpruner/PartitionSegmentPruner.java | 7 +-
.../routing/segmentpruner/SegmentPruner.java | 3 +-
.../segmentpruner/SegmentPrunerFactory.java | 50 +-
.../routing/segmentpruner/TimeSegmentPruner.java | 346 +++++++
.../routing/segmentpruner/interval/Interval.java | 104 +++
.../segmentpruner/interval/IntervalTree.java | 201 +++++
.../segmentselector/OfflineSegmentSelector.java | 8 +-
.../segmentselector/RealtimeSegmentSelector.java | 31 +-
.../routing/segmentselector/SegmentSelector.java | 3 +-
.../broker/routing/IntervalST/IntervalTest.java | 57 ++
.../routing/IntervalST/IntervalTreeTest.java | 89 ++
.../routing/segmentpruner/SegmentPrunerTest.java | 285 +++++-
.../common/function/scalar/JsonFunctions.java | 63 +-
.../apache/pinot/common/minion/MinionClient.java | 17 +-
.../common/minion/MinionRequestURLBuilder.java | 14 +-
.../pinot/common/function/JsonFunctionsTest.java | 3 +
.../pinot/common/minion/MinionClientTest.java | 2 +-
pinot-controller/pom.xml | 22 +
.../resources/PinotIngestionRestletResource.java | 194 ++++
.../api/resources/PinotTaskRestletResource.java | 16 +-
.../helix/ControllerRequestURLBuilder.java | 35 +
.../helix/core/PinotHelixResourceManager.java | 5 +-
.../helix/core/minion/PinotTaskManager.java | 169 ++--
.../pinot/controller/util/FileIngestionHelper.java | 147 +++
.../pinot/controller/util/FileIngestionUtils.java | 178 ++++
.../pinot/controller/ControllerTestSetup.java | 51 ++
...ontrollerTest.java => ControllerTestUtils.java} | 428 +++++----
.../controller/LeadControllerManagerTest.java | 2 +-
.../pinot/controller/api/AccessControlTest.java | 32 +-
...> PinotBrokerRestletResourceStatelessTest.java} | 2 +-
.../pinot/controller/api/PinotFileUploadTest.java | 30 +-
.../api/PinotIngestionRestletResourceTest.java | 133 +++
...PinotInstanceAssignmentRestletResourceTest.java | 157 ++--
.../api/PinotInstanceRestletResourceTest.java | 95 +-
.../api/PinotSchemaRestletResourceTest.java | 44 +-
.../api/PinotSegmentRestletResourceTest.java | 41 +-
.../api/PinotTableRestletResourceTest.java | 194 ++--
.../api/PinotTenantRestletResourceTest.java | 46 +-
.../pinot/controller/api/TableViewsTest.java | 77 +-
.../controller/api/upload/ZKOperatorTest.java | 64 +-
.../helix/ControllerInstanceToggleTest.java | 79 +-
...ontrollerPeriodicTaskStarterStatelessTest.java} | 2 +-
.../controller/helix/ControllerSentinelTestV2.java | 76 +-
...est.java => ControllerTenantStatelessTest.java} | 2 +-
.../pinot/controller/helix/ControllerTest.java | 98 +-
.../pinot/controller/helix/HelixHelperTest.java | 19 +-
....java => PinotControllerModeStatelessTest.java} | 14 +-
.../controller/helix/PinotResourceManagerTest.java | 76 +-
.../pinot/controller/helix/TableCacheTest.java | 119 +--
.../PinotHelixResourceManagerStatelessTest.java | 227 +++++
.../helix/core/PinotHelixResourceManagerTest.java | 403 ++++-----
...fflineNonReplicaGroupSegmentAssignmentTest.java | 22 +-
.../PinotLLCRealtimeSegmentManagerTest.java | 47 +-
.../helix/core/realtime/SegmentCompletionTest.java | 502 +++++++----
...va => TableRebalancerClusterStatelessTest.java} | 2 +-
.../core/retention/SegmentLineageCleanupTest.java | 77 +-
.../validation/ValidationManagerStatelessTest.java | 120 +++
.../validation/ValidationManagerTest.java | 105 +--
pinot-controller/testng-statefull.xml | 84 ++
pinot-controller/testng-stateless.xml | 42 +
.../org/apache/pinot/core/common/DataSource.java | 7 +
.../core/common/datatable/DataTableUtils.java | 95 +-
.../core/data/manager/SegmentDataManager.java | 2 +-
.../manager/offline/DimensionTableDataManager.java | 164 ++++
.../manager/offline/TableDataManagerProvider.java | 7 +-
.../realtime/HLRealtimeSegmentDataManager.java | 4 +-
.../realtime/LLRealtimeSegmentDataManager.java | 9 +-
.../generator/SegmentGeneratorConfig.java | 39 +-
.../indexsegment/mutable/MutableSegmentImpl.java | 9 +-
.../operator/filter/BitmapBasedFilterOperator.java | 6 +-
.../core/operator/filter/FilterOperatorUtils.java | 6 +
.../operator/filter/JsonMatchFilterOperator.java | 8 +-
.../FSTBasedRegexpPredicateEvaluatorFactory.java | 156 ++++
.../core/periodictask/PeriodicTaskScheduler.java | 12 +
.../org/apache/pinot/core/plan/FilterPlanNode.java | 37 +-
.../query/selection/SelectionOperatorUtils.java | 2 +-
.../converter/RealtimeSegmentConverter.java | 7 +-
.../RealtimeSegmentSegmentCreationDataSource.java | 3 +-
.../core/realtime/impl/RealtimeSegmentConfig.java | 24 +-
.../RealtimeLuceneTextIndexReader.java | 6 +
.../pinot/core/segment/creator/TextIndexType.java | 3 +-
.../creator/impl/SegmentColumnarIndexCreator.java | 34 +-
.../core/segment/creator/impl/V1Constants.java | 2 +
.../impl/inv/text/LuceneFSTIndexCreator.java | 103 +++
.../segment/index/column/ColumnIndexContainer.java | 5 +
.../index/column/PhysicalColumnIndexContainer.java | 37 +-
.../converter/SegmentV1V2ToV3FormatConverter.java | 39 +-
.../segment/index/datasource/BaseDataSource.java | 10 +-
.../index/datasource/ImmutableDataSource.java | 5 +-
.../index/datasource/MutableDataSource.java | 14 +-
.../segment/index/loader/IndexLoadingConfig.java | 22 +
.../segment/index/loader/SegmentPreProcessor.java | 10 +-
.../defaultcolumn/BaseDefaultColumnHandler.java | 2 +-
.../invertedindex/LuceneFSTIndexHandler.java | 157 ++++
.../segment/index/metadata/ColumnMetadata.java | 20 +-
.../index/readers/LuceneFSTIndexReader.java | 82 ++
.../segment/index/readers/TextIndexReader.java | 8 +
.../index/readers/text/LuceneTextIndexReader.java | 6 +
.../pinot/core/segment/store/ColumnIndexType.java | 1 +
.../core/segment/store/FilePerIndexDirectory.java | 9 +-
.../core/segment/store/SegmentDirectoryPaths.java | 9 +
.../segment/store/SingleFileIndexDirectory.java | 3 +
.../virtualcolumn/VirtualColumnIndexContainer.java | 5 +
.../core/startree/v2/store/StarTreeDataSource.java | 4 +-
.../apache/pinot/core/util/TableConfigUtils.java | 36 +-
.../org/apache/pinot/core/util/fst/FSTBuilder.java | 65 ++
.../pinot/core/util/fst/PinotBufferIndexInput.java | 89 ++
.../apache/pinot/core/util/fst/RegexpMatcher.java | 164 ++++
.../core/common/datatable/DataTableUtilsTest.java | 28 +-
.../offline/DimensionTableDataManagerTest.java | 173 ++++
.../index/creator/LuceneFSTIndexCreatorTest.java | 74 ++
.../core/segment/index/loader/LoaderTest.java | 139 +++
.../index/loader/SegmentPreProcessorTest.java | 105 ++-
.../pinot/core/util/TableConfigUtilsTest.java | 132 +++
.../queries/FSTBasedRegexpLikeQueriesTest.java | 477 ++++++++++
.../pinot/queries/TextSearchQueriesTest.java | 3 +-
.../java/org/apache/pinot/util/FSTBuilderTest.java | 107 +++
.../src/test/resources/data/dimBaseballTeams.avro | Bin 0 -> 2214 bytes
.../src/test/resources/data/dimBaseballTeams.avsc | 8 +
.../src/test/resources/data/dimBaseballTeams.csv | 52 ++
.../resources/data/dimBaseballTeams_schema.json | 14 +
.../src/test/resources/data/newColumnsSchema4.json | 84 ++
.../pinot/compat/tests/ClusterDescriptor.java | 7 +-
.../org/apache/pinot/compat/tests/TableOp.java | 55 +-
...vertToRawIndexMinionClusterIntegrationTest.java | 6 +-
.../tests/HybridClusterIntegrationTest.java | 2 +-
...fflineSegmentsMinionClusterIntegrationTest.java | 6 +-
.../tests/SimpleMinionClusterIntegrationTest.java | 17 +-
.../batch/common/SegmentGenerationUtilsTest.java | 16 +
.../standalone/SegmentGenerationJobRunner.java | 10 +-
.../standalone/SegmentGenerationJobRunnerTest.java | 54 ++
.../src/test/resources/log4j2.xml | 0
.../inputformat/avro/AvroRecordExtractor.java | 6 +-
.../plugin/inputformat/csv/CSVRecordExtractor.java | 3 +-
.../inputformat/json/JSONRecordExtractor.java | 6 +-
.../protobuf/ProtoBufRecordExtractor.java | 6 +-
.../inputformat/thrift/ThriftRecordExtractor.java | 6 +-
.../apache/pinot/spi/config/table/FieldConfig.java | 2 +-
.../pinot/spi/config/table/RoutingConfig.java | 1 +
.../spi/data/readers/RecordReaderFactory.java | 16 +
.../pinot/spi/utils/IngestionConfigUtils.java | 7 +
.../org/apache/pinot/tools/BootstrapTableTool.java | 16 +-
.../org/apache/pinot/tools/JoinQuickStart.java | 124 +++
.../pinot/tools/admin/PinotAdministrator.java | 18 +-
.../tools/admin/command/QuickstartRunner.java | 27 +-
pom.xml | 4 +-
thirdeye/pom.xml | 4 +-
thirdeye/thirdeye-frontend/.prettierrc | 8 +
.../app/pods/components/alert-details/component.js | 9 +-
.../pods/components/anomaly-summary/component.js | 259 +++---
.../pods/components/breadcrumb-list/component.js | 56 ++
.../pods/components/breadcrumb-list/template.hbs | 10 +
.../components/composite-anomalies/component.js | 87 ++
.../parent-anomalies/component.js | 117 +++
.../parent-anomalies/template.hbs | 20 +
.../components/composite-anomalies/template.hbs | 8 +
.../app/pods/components/stats-cards/template.hbs | 43 +-
.../anomalies-list/template.hbs | 3 +
.../resolution/component.js | 15 +
.../resolution/template.hbs | 11 +
.../start-duration/template.hbs | 8 +
.../explore/composite-anomalies/controller.js | 97 +-
.../manage/explore/composite-anomalies/route.js | 6 +-
.../explore/composite-anomalies/template.hbs | 47 +-
thirdeye/thirdeye-frontend/app/styles/app.scss | 2 +
.../app/styles/components/breadcrumb-list.scss | 19 +
.../styles/pods/custom/parent-anomalies-table.scss | 21 +
.../app/styles/shared/_styles.scss | 10 +
thirdeye/thirdeye-frontend/app/utils/pub-sub.js | 72 ++
thirdeye/thirdeye-frontend/package.json | 7 +-
.../components/breadcrumb-list/component-test.js | 33 +
.../composite-anomalies/component-test.js | 20 +
.../parent-anomalies/component-test.js | 69 ++
.../pods/components/stats-cards/component-test.js | 76 +-
.../tests/unit/utils/pub-sub-test.js | 19 +
thirdeye/thirdeye-frontend/yarn.lock | 996 ++++++++++++++++++++-
thirdeye/thirdeye-pinot/pom.xml | 2 +-
183 files changed, 9105 insertions(+), 1985 deletions(-)
create mode 100644 pinot-broker/src/main/java/org/apache/pinot/broker/routing/segmentpruner/TimeSegmentPruner.java
create mode 100644 pinot-broker/src/main/java/org/apache/pinot/broker/routing/segmentpruner/interval/Interval.java
create mode 100644 pinot-broker/src/main/java/org/apache/pinot/broker/routing/segmentpruner/interval/IntervalTree.java
create mode 100644 pinot-broker/src/test/java/org/apache/pinot/broker/routing/IntervalST/IntervalTest.java
create mode 100644 pinot-broker/src/test/java/org/apache/pinot/broker/routing/IntervalST/IntervalTreeTest.java
create mode 100644 pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotIngestionRestletResource.java
create mode 100644 pinot-controller/src/main/java/org/apache/pinot/controller/util/FileIngestionHelper.java
create mode 100644 pinot-controller/src/main/java/org/apache/pinot/controller/util/FileIngestionUtils.java
create mode 100644 pinot-controller/src/test/java/org/apache/pinot/controller/ControllerTestSetup.java
copy pinot-controller/src/test/java/org/apache/pinot/controller/{helix/ControllerTest.java => ControllerTestUtils.java} (62%)
rename pinot-controller/src/test/java/org/apache/pinot/controller/api/{PinotBrokerRestletResourceTest.java => PinotBrokerRestletResourceStatelessTest.java} (98%)
create mode 100644 pinot-controller/src/test/java/org/apache/pinot/controller/api/PinotIngestionRestletResourceTest.java
rename pinot-controller/src/test/java/org/apache/pinot/controller/helix/{ControllerPeriodicTaskStarterTest.java => ControllerPeriodicTaskStarterStatelessTest.java} (97%)
rename pinot-controller/src/test/java/org/apache/pinot/controller/helix/{ControllerTenantTest.java => ControllerTenantStatelessTest.java} (99%)
rename pinot-controller/src/test/java/org/apache/pinot/controller/helix/{PinotControllerModeTest.java => PinotControllerModeStatelessTest.java} (99%)
create mode 100644 pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/PinotHelixResourceManagerStatelessTest.java
rename pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/rebalance/{TableRebalancerClusterTest.java => TableRebalancerClusterStatelessTest.java} (99%)
create mode 100644 pinot-controller/src/test/java/org/apache/pinot/controller/validation/ValidationManagerStatelessTest.java
create mode 100644 pinot-controller/testng-statefull.xml
create mode 100644 pinot-controller/testng-stateless.xml
create mode 100644 pinot-core/src/main/java/org/apache/pinot/core/data/manager/offline/DimensionTableDataManager.java
create mode 100644 pinot-core/src/main/java/org/apache/pinot/core/operator/filter/predicate/FSTBasedRegexpPredicateEvaluatorFactory.java
create mode 100644 pinot-core/src/main/java/org/apache/pinot/core/segment/creator/impl/inv/text/LuceneFSTIndexCreator.java
create mode 100644 pinot-core/src/main/java/org/apache/pinot/core/segment/index/loader/invertedindex/LuceneFSTIndexHandler.java
create mode 100644 pinot-core/src/main/java/org/apache/pinot/core/segment/index/readers/LuceneFSTIndexReader.java
create mode 100644 pinot-core/src/main/java/org/apache/pinot/core/util/fst/FSTBuilder.java
create mode 100644 pinot-core/src/main/java/org/apache/pinot/core/util/fst/PinotBufferIndexInput.java
create mode 100644 pinot-core/src/main/java/org/apache/pinot/core/util/fst/RegexpMatcher.java
create mode 100644 pinot-core/src/test/java/org/apache/pinot/core/data/manager/offline/DimensionTableDataManagerTest.java
create mode 100644 pinot-core/src/test/java/org/apache/pinot/core/segment/index/creator/LuceneFSTIndexCreatorTest.java
create mode 100644 pinot-core/src/test/java/org/apache/pinot/queries/FSTBasedRegexpLikeQueriesTest.java
create mode 100644 pinot-core/src/test/java/org/apache/pinot/util/FSTBuilderTest.java
create mode 100644 pinot-core/src/test/resources/data/dimBaseballTeams.avro
create mode 100644 pinot-core/src/test/resources/data/dimBaseballTeams.avsc
create mode 100644 pinot-core/src/test/resources/data/dimBaseballTeams.csv
create mode 100644 pinot-core/src/test/resources/data/dimBaseballTeams_schema.json
create mode 100644 pinot-core/src/test/resources/data/newColumnsSchema4.json
copy pinot-core/src/main/java/org/apache/pinot/core/segment/creator/TextIndexType.java => pinot-integration-tests/src/test/java/org/apache/pinot/compat/tests/ClusterDescriptor.java (85%)
create mode 100644 pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-standalone/src/test/java/org/apache/pinot/plugin/ingestion/batch/standalone/SegmentGenerationJobRunnerTest.java
copy {pinot-tools => pinot-plugins/pinot-batch-ingestion/pinot-batch-ingestion-standalone}/src/test/resources/log4j2.xml (100%)
create mode 100644 pinot-tools/src/main/java/org/apache/pinot/tools/JoinQuickStart.java
create mode 100644 thirdeye/thirdeye-frontend/app/pods/components/breadcrumb-list/component.js
create mode 100644 thirdeye/thirdeye-frontend/app/pods/components/breadcrumb-list/template.hbs
create mode 100644 thirdeye/thirdeye-frontend/app/pods/components/composite-anomalies/component.js
create mode 100644 thirdeye/thirdeye-frontend/app/pods/components/composite-anomalies/parent-anomalies/component.js
create mode 100644 thirdeye/thirdeye-frontend/app/pods/components/composite-anomalies/parent-anomalies/template.hbs
create mode 100644 thirdeye/thirdeye-frontend/app/pods/components/composite-anomalies/template.hbs
create mode 100644 thirdeye/thirdeye-frontend/app/pods/custom/composite-anomalies-table/anomalies-list/template.hbs
create mode 100644 thirdeye/thirdeye-frontend/app/pods/custom/composite-anomalies-table/resolution/component.js
create mode 100644 thirdeye/thirdeye-frontend/app/pods/custom/composite-anomalies-table/resolution/template.hbs
create mode 100644 thirdeye/thirdeye-frontend/app/pods/custom/composite-anomalies-table/start-duration/template.hbs
create mode 100644 thirdeye/thirdeye-frontend/app/styles/components/breadcrumb-list.scss
create mode 100644 thirdeye/thirdeye-frontend/app/styles/pods/custom/parent-anomalies-table.scss
create mode 100644 thirdeye/thirdeye-frontend/app/utils/pub-sub.js
create mode 100644 thirdeye/thirdeye-frontend/tests/integration/pods/components/breadcrumb-list/component-test.js
create mode 100644 thirdeye/thirdeye-frontend/tests/integration/pods/components/composite-anomalies/component-test.js
create mode 100644 thirdeye/thirdeye-frontend/tests/integration/pods/components/composite-anomalies/parent-anomalies/component-test.js
create mode 100644 thirdeye/thirdeye-frontend/tests/unit/utils/pub-sub-test.js
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org