You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by xi...@apache.org on 2020/12/29 01:41:20 UTC
[incubator-pinot] branch
adding_pinot_minion_segment_creation_tasks_2 updated (5ec925d -> 9b3d2bf)
This is an automated email from the ASF dual-hosted git repository.
xiangfu pushed a change to branch adding_pinot_minion_segment_creation_tasks_2
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git.
discard 5ec925d Address comments
discard 55af63b Adding logic to infer recordreader classes from input format
discard 8aa62e4 Adding pinot minion segment creation task
add 85cf696 Fixing the issue for realtime table creation with env variables in stream configs (#6373)
add 10dad7d Ingestion resource with APIs for ingestion via file/URI (#6354)
add b8bc74f compatibility test: create/delete table operations (#6360)
add 7580ae4 [TE]frontend - Integrate Performance Stats into Composite Anomalies (#6366)
add 33de6dc Introduce DimensionTableDataManager (#6346)
add 6d8b09e Update Helm readme with latest incubator repo (#6381)
add 1bb6c14 Add FST index which works on top of REGEXP_LIKE operator. (#6120)
add fed6750 Adding a version option to pinot admin to show all the component versions (#6380)
add e268a79 Adding S3PinotFS as one of the default PinotFS for Quickstart (#6379)
add ea0bfa0 Add json index support (#6216)
add 9580d83 Adding pinot minion segment creation task
add 6b16b4b Adding logic to infer recordreader classes from input format
add 9b3d2bf Address comments
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (5ec925d)
\
N -- N -- N refs/heads/adding_pinot_minion_segment_creation_tasks_2 (9b3d2bf)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
No new revisions were added by this update.
Summary of changes:
.gitignore | 1 -
kubernetes/helm/README.md | 4 +-
.../antlr4/org/apache/pinot/pql/parsers/PQL2.g4 | 4 +
.../pinot/common/request/AggregationInfo.java | 52 +-
.../apache/pinot/common/request/BrokerRequest.java | 353 +-
.../apache/pinot/common/request/DataSource.java | 11 +-
.../apache/pinot/common/request/Expression.java | 35 +-
.../pinot/common/request/ExpressionType.java | 4 +-
.../pinot/common/request/FilterOperator.java | 11 +-
.../apache/pinot/common/request/FilterQuery.java | 46 +-
.../pinot/common/request/FilterQueryMap.java | 11 +-
.../org/apache/pinot/common/request/Function.java | 21 +-
.../org/apache/pinot/common/request/GroupBy.java | 85 +-
.../pinot/common/request/HavingFilterQuery.java | 46 +-
.../pinot/common/request/HavingFilterQueryMap.java | 11 +-
.../apache/pinot/common/request/Identifier.java | 16 +-
.../pinot/common/request/InstanceRequest.java | 76 +-
.../org/apache/pinot/common/request/Literal.java | 4 +-
.../apache/pinot/common/request/PinotQuery.java | 61 +-
.../apache/pinot/common/request/QuerySource.java | 11 +-
.../org/apache/pinot/common/request/QueryType.java | 31 +-
.../org/apache/pinot/common/request/Selection.java | 94 +-
.../apache/pinot/common/request/SelectionSort.java | 16 +-
.../pinot/common/response/ProcessingException.java | 283 +-
.../apache/pinot/parsers/utils/ParserUtils.java | 1 +
.../parsers/PinotQuery2BrokerRequestConverter.java | 3 +-
.../pinot/pql/parsers/pql2/ast/FilterKind.java | 3 +-
.../apache/pinot/sql/parsers/CalciteSqlParser.java | 21 +-
pinot-common/src/thrift/request.thrift | 3 +-
pinot-controller/pom.xml | 6 +
.../resources/PinotIngestionRestletResource.java | 194 +
.../helix/ControllerRequestURLBuilder.java | 35 +
.../helix/core/PinotHelixResourceManager.java | 5 +-
.../pinot/controller/util/FileIngestionHelper.java | 147 +
.../pinot/controller/util/FileIngestionUtils.java | 178 +
.../api/PinotIngestionRestletResourceTest.java | 133 +
.../org/apache/pinot/core/common/DataSource.java | 14 +
.../core/data/manager/SegmentDataManager.java | 2 +-
.../manager/offline/DimensionTableDataManager.java | 164 +
.../manager/offline/TableDataManagerProvider.java | 7 +-
.../realtime/HLRealtimeSegmentDataManager.java | 4 +-
.../realtime/LLRealtimeSegmentDataManager.java | 9 +-
.../generator/SegmentGeneratorConfig.java | 56 +-
.../indexsegment/mutable/MutableSegmentImpl.java | 30 +-
.../io/util/VarLengthBytesValueReaderWriter.java | 241 -
.../pinot/core/io/util/VarLengthValueReader.java | 121 +
.../pinot/core/io/util/VarLengthValueWriter.java | 133 +
.../operator/filter/BitmapBasedFilterOperator.java | 6 +-
.../core/operator/filter/FilterOperatorUtils.java | 15 +-
.../operator/filter/JsonMatchFilterOperator.java | 59 +
.../FSTBasedRegexpPredicateEvaluatorFactory.java | 156 +
.../org/apache/pinot/core/plan/FilterPlanNode.java | 38 +-
.../{Predicate.java => JsonMatchPredicate.java} | 57 +-
.../query/request/context/predicate/Predicate.java | 2 +-
.../context/utils/QueryContextConverterUtils.java | 4 +
.../converter/RealtimeSegmentConverter.java | 7 +-
.../RealtimeSegmentSegmentCreationDataSource.java | 3 +-
.../core/realtime/impl/RealtimeSegmentConfig.java | 24 +-
.../RealtimeLuceneTextIndexReader.java | 6 +
.../{TextIndexType.java => JsonIndexCreator.java} | 24 +-
.../pinot/core/segment/creator/SegmentCreator.java | 3 +-
.../pinot/core/segment/creator/TextIndexType.java | 3 +-
.../creator/impl/SegmentColumnarIndexCreator.java | 100 +-
.../creator/impl/SegmentDictionaryCreator.java | 41 +-
.../core/segment/creator/impl/V1Constants.java | 4 +
.../impl/inv/BitmapInvertedIndexWriter.java | 90 +
.../inv/OffHeapBitmapInvertedIndexCreator.java | 43 +-
.../impl/inv/OnHeapBitmapInvertedIndexCreator.java | 45 +-
.../impl/inv/json/BaseJsonIndexCreator.java | 173 +
.../impl/inv/json/OffHeapJsonIndexCreator.java | 282 +
.../impl/inv/json/OnHeapJsonIndexCreator.java | 60 +
.../impl/inv/text/LuceneFSTIndexCreator.java | 103 +
.../segment/index/column/ColumnIndexContainer.java | 17 +-
.../index/column/PhysicalColumnIndexContainer.java | 56 +-
.../converter/SegmentV1V2ToV3FormatConverter.java | 53 +-
.../segment/index/datasource/BaseDataSource.java | 20 +-
.../index/datasource/ImmutableDataSource.java | 5 +-
.../index/datasource/MutableDataSource.java | 15 +-
.../segment/index/loader/IndexLoadingConfig.java | 32 +
.../segment/index/loader/SegmentPreProcessor.java | 17 +-
.../defaultcolumn/BaseDefaultColumnHandler.java | 3 +-
.../loader/invertedindex/JsonIndexHandler.java | 176 +
.../invertedindex/LuceneFSTIndexHandler.java | 157 +
.../segment/index/metadata/ColumnMetadata.java | 20 +-
.../index/readers/BaseImmutableDictionary.java | 15 +-
.../segment/index/readers/JsonIndexReader.java | 243 +
.../index/readers/LuceneFSTIndexReader.java | 82 +
.../segment/index/readers/TextIndexReader.java | 8 +
.../index/readers/text/LuceneTextIndexReader.java | 6 +
.../pinot/core/segment/store/ColumnIndexType.java | 2 +
.../core/segment/store/FilePerIndexDirectory.java | 13 +-
.../pinot/core/segment/store/SegmentDirectory.java | 25 +-
.../core/segment/store/SegmentDirectoryPaths.java | 9 +
.../segment/store/SegmentLocalFSDirectory.java | 10 -
.../segment/store/SingleFileIndexDirectory.java | 3 +
.../virtualcolumn/VirtualColumnIndexContainer.java | 11 +
.../core/startree/v2/store/StarTreeDataSource.java | 5 +-
.../apache/pinot/core/util/TableConfigUtils.java | 36 +-
.../org/apache/pinot/core/util/fst/FSTBuilder.java | 65 +
.../pinot/core/util/fst/PinotBufferIndexInput.java | 89 +
.../apache/pinot/core/util/fst/RegexpMatcher.java | 164 +
.../offline/DimensionTableDataManagerTest.java | 173 +
.../util/VarLengthBytesValueReaderWriterTest.java | 172 -
.../io/util/VarLengthValueReaderWriterTest.java | 117 +
.../pinot/core/segment/index/JsonIndexTest.java | 205 +
.../index/creator/LuceneFSTIndexCreatorTest.java | 74 +
.../core/segment/index/loader/LoaderTest.java | 139 +
.../index/loader/SegmentPreProcessorTest.java | 105 +-
.../segment/store/SegmentLocalFSDirectoryTest.java | 2 +-
.../pinot/core/util/TableConfigUtilsTest.java | 132 +
.../queries/FSTBasedRegexpLikeQueriesTest.java | 477 +
.../pinot/queries/TextSearchQueriesTest.java | 3 +-
.../java/org/apache/pinot/util/FSTBuilderTest.java | 107 +
.../src/test/resources/data/dimBaseballTeams.avro | Bin 0 -> 2214 bytes
.../src/test/resources/data/dimBaseballTeams.avsc | 8 +
.../src/test/resources/data/dimBaseballTeams.csv | 52 +
.../resources/data/dimBaseballTeams_schema.json | 14 +
.../src/test/resources/data/newColumnsSchema4.json | 84 +
.../pinot/compat/tests/ClusterDescriptor.java | 7 +-
.../org/apache/pinot/compat/tests/TableOp.java | 55 +-
.../apache/pinot/spi/config/table/FieldConfig.java | 2 +-
.../pinot/spi/config/table/IndexingConfig.java | 9 +
.../spi/data/readers/RecordReaderFactory.java | 16 +
.../pinot/spi/utils/IngestionConfigUtils.java | 7 +
.../java/org/apache/pinot/spi/utils/JsonUtils.java | 191 +-
.../org/apache/pinot/spi/utils/JsonUtilsTest.java | 145 +
pinot-tools/pom.xml | 11 +
.../org/apache/pinot/tools/BootstrapTableTool.java | 8 +-
.../org/apache/pinot/tools/JoinQuickStart.java | 124 +
.../apache/pinot/tools/JsonIndexQuickStart.java | 89 +
.../pinot/tools/admin/PinotAdministrator.java | 18 +-
.../tools/admin/command/QuickstartRunner.java | 27 +-
.../githubEvents_offline_table_config.json | 27 +
.../batch/githubEvents/githubEvents_schema.json | 40 +
.../batch/githubEvents/ingestionJobSpec.yaml | 139 +
.../githubEvents/rawdata/githubEvents_data.json | 10000 +++++++++++++++++++
.../batch/githubEvents/sparkIngestionJobSpec.yaml | 147 +
thirdeye/thirdeye-frontend/.prettierrc | 8 +
.../explore/composite-anomalies/controller.js | 23 +-
.../explore/composite-anomalies/template.hbs | 53 +-
140 files changed, 17026 insertions(+), 1486 deletions(-)
create mode 100644 pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotIngestionRestletResource.java
create mode 100644 pinot-controller/src/main/java/org/apache/pinot/controller/util/FileIngestionHelper.java
create mode 100644 pinot-controller/src/main/java/org/apache/pinot/controller/util/FileIngestionUtils.java
create mode 100644 pinot-controller/src/test/java/org/apache/pinot/controller/api/PinotIngestionRestletResourceTest.java
create mode 100644 pinot-core/src/main/java/org/apache/pinot/core/data/manager/offline/DimensionTableDataManager.java
delete mode 100644 pinot-core/src/main/java/org/apache/pinot/core/io/util/VarLengthBytesValueReaderWriter.java
create mode 100644 pinot-core/src/main/java/org/apache/pinot/core/io/util/VarLengthValueReader.java
create mode 100644 pinot-core/src/main/java/org/apache/pinot/core/io/util/VarLengthValueWriter.java
create mode 100644 pinot-core/src/main/java/org/apache/pinot/core/operator/filter/JsonMatchFilterOperator.java
create mode 100644 pinot-core/src/main/java/org/apache/pinot/core/operator/filter/predicate/FSTBasedRegexpPredicateEvaluatorFactory.java
copy pinot-core/src/main/java/org/apache/pinot/core/query/request/context/predicate/{Predicate.java => JsonMatchPredicate.java} (50%)
copy pinot-core/src/main/java/org/apache/pinot/core/segment/creator/{TextIndexType.java => JsonIndexCreator.java} (69%)
create mode 100644 pinot-core/src/main/java/org/apache/pinot/core/segment/creator/impl/inv/BitmapInvertedIndexWriter.java
create mode 100644 pinot-core/src/main/java/org/apache/pinot/core/segment/creator/impl/inv/json/BaseJsonIndexCreator.java
create mode 100644 pinot-core/src/main/java/org/apache/pinot/core/segment/creator/impl/inv/json/OffHeapJsonIndexCreator.java
create mode 100644 pinot-core/src/main/java/org/apache/pinot/core/segment/creator/impl/inv/json/OnHeapJsonIndexCreator.java
create mode 100644 pinot-core/src/main/java/org/apache/pinot/core/segment/creator/impl/inv/text/LuceneFSTIndexCreator.java
create mode 100644 pinot-core/src/main/java/org/apache/pinot/core/segment/index/loader/invertedindex/JsonIndexHandler.java
create mode 100644 pinot-core/src/main/java/org/apache/pinot/core/segment/index/loader/invertedindex/LuceneFSTIndexHandler.java
create mode 100644 pinot-core/src/main/java/org/apache/pinot/core/segment/index/readers/JsonIndexReader.java
create mode 100644 pinot-core/src/main/java/org/apache/pinot/core/segment/index/readers/LuceneFSTIndexReader.java
create mode 100644 pinot-core/src/main/java/org/apache/pinot/core/util/fst/FSTBuilder.java
create mode 100644 pinot-core/src/main/java/org/apache/pinot/core/util/fst/PinotBufferIndexInput.java
create mode 100644 pinot-core/src/main/java/org/apache/pinot/core/util/fst/RegexpMatcher.java
create mode 100644 pinot-core/src/test/java/org/apache/pinot/core/data/manager/offline/DimensionTableDataManagerTest.java
delete mode 100644 pinot-core/src/test/java/org/apache/pinot/core/io/util/VarLengthBytesValueReaderWriterTest.java
create mode 100644 pinot-core/src/test/java/org/apache/pinot/core/io/util/VarLengthValueReaderWriterTest.java
create mode 100644 pinot-core/src/test/java/org/apache/pinot/core/segment/index/JsonIndexTest.java
create mode 100644 pinot-core/src/test/java/org/apache/pinot/core/segment/index/creator/LuceneFSTIndexCreatorTest.java
create mode 100644 pinot-core/src/test/java/org/apache/pinot/queries/FSTBasedRegexpLikeQueriesTest.java
create mode 100644 pinot-core/src/test/java/org/apache/pinot/util/FSTBuilderTest.java
create mode 100644 pinot-core/src/test/resources/data/dimBaseballTeams.avro
create mode 100644 pinot-core/src/test/resources/data/dimBaseballTeams.avsc
create mode 100644 pinot-core/src/test/resources/data/dimBaseballTeams.csv
create mode 100644 pinot-core/src/test/resources/data/dimBaseballTeams_schema.json
create mode 100644 pinot-core/src/test/resources/data/newColumnsSchema4.json
copy pinot-core/src/main/java/org/apache/pinot/core/segment/creator/TextIndexType.java => pinot-integration-tests/src/test/java/org/apache/pinot/compat/tests/ClusterDescriptor.java (85%)
create mode 100644 pinot-spi/src/test/java/org/apache/pinot/spi/utils/JsonUtilsTest.java
create mode 100644 pinot-tools/src/main/java/org/apache/pinot/tools/JoinQuickStart.java
create mode 100644 pinot-tools/src/main/java/org/apache/pinot/tools/JsonIndexQuickStart.java
create mode 100644 pinot-tools/src/main/resources/examples/batch/githubEvents/githubEvents_offline_table_config.json
create mode 100644 pinot-tools/src/main/resources/examples/batch/githubEvents/githubEvents_schema.json
create mode 100644 pinot-tools/src/main/resources/examples/batch/githubEvents/ingestionJobSpec.yaml
create mode 100644 pinot-tools/src/main/resources/examples/batch/githubEvents/rawdata/githubEvents_data.json
create mode 100644 pinot-tools/src/main/resources/examples/batch/githubEvents/sparkIngestionJobSpec.yaml
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org