You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by ki...@apache.org on 2020/11/24 19:49:03 UTC
[incubator-pinot] branch json-indexing updated (045f7ef -> bd6b9cc)
This is an automated email from the ASF dual-hosted git repository.
kishoreg pushed a change to branch json-indexing
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git.
discard 045f7ef Fixing table config
discard 64dba07 adding example data for json
discard e898efd changing table name from super to personJson
discard e9a8ad5 Adding index creator and reader
discard 8462646 Initial commit
add e3b0bfc scalar functions for array (#6105)
add bee125e showing Query Result as blank table and json format toggle even if query response has 0 records (#6223)
add d033a11 Add profile release-sign-artifacts for pinot-spark-connector (#6229)
add ca8545b [TE] frontend - abachuk/alert-details-ui-tweaks - formatting alert header (#6210)
add 2d446d2 [TE] Remove maven-shade-plugin and switch from org.reflections to io.classgraph (#6238)
add 4586279 [TE]frontend - Refactor detection-health and stats component (#6227)
add 8ce2271 Adding support of logical functions AND and OR (#6249)
add c8d7efc Make default operator for multi-term and phrase text search queries configurable (#6251)
add bd2a6ce Bug-fix: Fix logging potentialy null variable. (#6252)
add 750af31 Always read start/end time in millis from the segment ZK metadata (#6239)
add 5a53fbe Replace sql literal regex replace (#6258)
add 9eaea99 Add stream and batch to ingestionConfig (#6247)
add f40c2ac Set S3 Bucket ACL policy from config (#6272)
add 96647ad Allow setting HTTP headers and parameters during schema upload (#6270)
add c757f24 Update download page for release 0.6.0 (#6274)
add b009fd8 Improve comparison coverage for selection SQL queries in ClusterIntegrationTestUtils(#6193) (#6224)
add 2a04de8 [TE] rest-api enhance implementation to calculate alert performance (#6273)
add bd2905f Fix download link for 0.6.0 (#6278)
add f099515 Update snapshot version to 0.7.0-SNAPSHOT (#6279)
add 6d655f9 Adding config utils to apply environment variables and apply it to table config (#6271)
add fa7b0e4 Perf optimization for SQL GROUP BY ORDER BY (#6225)
add 4a6e094 add api for cluster manager to get table state (#6211)
add c4f8ec1 update superset source repo in superset docker image build script (#6283)
add 46212f4 Initial commit
add ff141fa Adding index creator and reader
add 8af3c11 changing table name from super to personJson
add 2a4b98f adding example data for json
add bd6b9cc adding support querying based on array index
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (045f7ef)
\
N -- N -- N refs/heads/json-indexing (bd6b9cc)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
No new revisions were added by this update.
Summary of changes:
docker/images/pinot-superset/Dockerfile | 5 +-
docker/images/pinot-superset/README.md | 6 +-
pinot-broker/pom.xml | 2 +-
.../requesthandler/BaseBrokerRequestHandler.java | 2 +-
.../routing/timeboundary/TimeBoundaryManager.java | 97 +++--
pinot-clients/pinot-java-client/pom.xml | 2 +-
pinot-clients/pinot-jdbc-client/pom.xml | 2 +-
pinot-clients/pom.xml | 2 +-
pinot-common/pom.xml | 2 +-
.../pinot/common/function/FunctionInvoker.java | 19 +-
.../pinot/common/function/FunctionUtils.java | 32 ++
.../common/function/TransformFunctionType.java | 3 +
.../common/function/scalar/ArrayFunctions.java | 80 ++++
.../pinot/common/metadata/ZKMetadataProvider.java | 5 +-
.../common/metadata/segment/SegmentZKMetadata.java | 291 +++++++--------
.../apache/pinot/common/metrics/BrokerMeter.java | 3 +
.../apache/pinot/common/metrics/ServerMeter.java | 2 +
.../common/tier/TimeBasedTierSegmentSelector.java | 7 +-
.../apache/pinot/common/utils/CommonConstants.java | 16 +-
.../org/apache/pinot/common/utils/DataTable.java | 2 +
.../common/utils/FileUploadDownloadClient.java | 25 +-
.../apache/pinot/common/utils/PinotDataType.java | 32 +-
.../common/utils/config/TableConfigUtils.java | 2 +-
.../pinot/common/utils/request/RequestUtils.java | 2 +-
.../pinot/common/data/DateTimeFormatSpecTest.java | 70 ++--
.../common/utils/config/TableConfigSerDeTest.java | 27 +-
pinot-connectors/pinot-spark-connector/pom.xml | 17 +-
pinot-connectors/pom.xml | 2 +-
pinot-controller/pom.xml | 2 +-
.../api/resources/PinotTableRestletResource.java | 27 ++
.../helix/core/PinotHelixResourceManager.java | 30 +-
.../helix/core/PinotTableIdealStateBuilder.java | 3 +-
.../RealtimeToOfflineSegmentsTaskGenerator.java | 26 +-
.../realtime/PinotLLCRealtimeSegmentManager.java | 9 +-
.../core/realtime/PinotRealtimeSegmentManager.java | 7 +-
.../core/realtime/SegmentCompletionManager.java | 5 +-
.../helix/core/rebalance/TableRebalancer.java | 3 +-
.../helix/core/relocation/SegmentRelocator.java | 4 +-
.../helix/core/retention/RetentionManager.java | 3 +-
.../retention/strategy/TimeRetentionStrategy.java | 13 +-
.../controller/util/TableRetentionValidator.java | 23 +-
.../validation/OfflineSegmentIntervalChecker.java | 18 +-
.../RealtimeSegmentValidationManager.java | 6 +-
.../src/main/resources/app/pages/Query.tsx | 25 +-
.../main/resources/app/utils/PinotMethodUtils.ts | 2 +-
.../api/PinotTableRestletResourceTest.java | 44 +++
.../PinotLLCRealtimeSegmentManagerTest.java | 19 +-
pinot-core/pom.xml | 2 +-
.../realtime/HLRealtimeSegmentDataManager.java | 3 +-
.../realtime/LLRealtimeSegmentDataManager.java | 4 +-
.../core/data/table/ConcurrentIndexedTable.java | 74 ++--
.../apache/pinot/core/data/table/IndexedTable.java | 49 +--
.../pinot/core/data/table/SimpleIndexedTable.java | 57 ++-
.../apache/pinot/core/data/table/TableResizer.java | 90 ++---
.../table/UnboundedConcurrentIndexedTable.java | 80 ++++
.../operator/blocks/IntermediateResultsBlock.java | 20 +-
.../combine/GroupByOrderByCombineOperator.java | 27 +-
...tion.java => AndOperatorTransformFunction.java} | 25 +-
.../function/LogicalOperatorTransformFunction.java | 79 ++++
...ction.java => OrOperatorTransformFunction.java} | 25 +-
.../function/ScalarTransformFunctionWrapper.java | 122 +++++-
.../function/TransformFunctionFactory.java | 5 +
.../apache/pinot/core/plan/CombinePlanNode.java | 10 +-
.../org/apache/pinot/core/plan/FilterPlanNode.java | 2 +-
.../core/plan/maker/InstancePlanMakerImplV2.java | 16 +-
.../core/query/reduce/BrokerReduceService.java | 8 +-
.../core/query/reduce/DataTableReducerContext.java | 11 +-
.../core/query/reduce/GroupByDataTableReducer.java | 37 +-
.../pinot/core/query/scheduler/QueryScheduler.java | 10 +
.../segment/creator/impl/inv/JSONIndexCreator.java | 215 ++++-------
.../segment/index/loader/IndexLoadingConfig.java | 4 +
.../segment/index/readers/JSONIndexReader.java | 17 +-
.../index/readers/text/LuceneTextIndexReader.java | 8 +
.../org/apache/pinot/core/util/ArrayCopyUtils.java | 6 +
.../org/apache/pinot/core/util/IngestionUtils.java | 5 +-
.../apache/pinot/core/util/ReplicationUtils.java | 5 +-
.../apache/pinot/core/util/TableConfigUtils.java | 49 ++-
.../core/data/function/InbuiltFunctionsTest.java | 67 ++++
.../ExpressionTransformerTest.java | 10 +-
.../recordtransformer/RecordTransformerTest.java | 18 +-
.../pinot/core/data/table/IndexedTableTest.java | 25 +-
.../pinot/core/data/table/TableResizerTest.java | 26 +-
.../operator/combine/CombineSlowOperatorsTest.java | 2 +-
.../combine/SelectionCombineOperatorTest.java | 3 +-
.../function/AndOperatorTransformFunctionTest.java | 20 +-
.../function/BaseTransformFunctionTest.java | 47 ++-
.../LogicalOperatorTransformFunctionTest.java | 81 ++++
.../function/OrOperatorTransformFunctionTest.java | 20 +-
.../ScalarTransformFunctionWrapperTest.java | 248 ++++++++++--
.../pinot/core/plan/CombinePlanNodeTest.java | 9 +-
.../SegmentGenerationWithFilterRecordsTest.java | 10 +-
.../apache/pinot/core/util/IngestionUtilsTest.java | 41 +-
.../apache/pinot/core/util/SchemaUtilsTest.java | 6 +-
.../pinot/core/util/TableConfigUtilsTest.java | 120 ++++--
.../pinot/queries/TextSearchQueriesTest.java | 49 ++-
pinot-distribution/pom.xml | 2 +-
pinot-integration-tests/pom.xml | 2 +-
.../tests/BaseClusterIntegrationTest.java | 49 +--
.../tests/ClusterIntegrationTestUtils.java | 415 +++++++++++----------
.../IngestionConfigHybridIntegrationTest.java | 17 +-
.../LuceneRealtimeClusterIntegrationTest.java | 1 -
.../tests/MapTypeClusterIntegrationTest.java | 7 +-
.../tests/OfflineClusterIntegrationTest.java | 28 ++
...rformance_2014_100k_subset.test_queries_500.sql | 10 +-
pinot-minion/pom.xml | 2 +-
.../RealtimeToOfflineSegmentsTaskExecutor.java | 16 +-
.../RealtimeToOfflineSegmentsTaskExecutorTest.java | 6 +-
pinot-perf/pom.xml | 2 +-
.../apache/pinot/perf/BenchmarkCombineGroupBy.java | 6 +-
.../apache/pinot/perf/BenchmarkIndexedTable.java | 7 +-
.../pinot-batch-ingestion-common/pom.xml | 2 +-
.../batch/common/SegmentGenerationTaskRunner.java | 8 +-
.../pinot-batch-ingestion-hadoop/pom.xml | 2 +-
.../pinot-batch-ingestion-spark/pom.xml | 2 +-
.../pinot-batch-ingestion-standalone/pom.xml | 2 +-
pinot-plugins/pinot-batch-ingestion/pom.xml | 2 +-
.../v0_deprecated/pinot-hadoop/pom.xml | 2 +-
.../pinot/hadoop/job/HadoopSegmentCreationJob.java | 3 +-
.../hadoop/job/HadoopSegmentPreprocessingJob.java | 11 +-
.../hadoop/job/mappers/SegmentCreationMapper.java | 6 +-
.../v0_deprecated/pinot-ingestion-common/pom.xml | 2 +-
.../v0_deprecated/pinot-spark/pom.xml | 2 +-
.../spark/jobs/SparkSegmentCreationFunction.java | 4 +-
.../pinot/spark/jobs/SparkSegmentCreationJob.java | 3 +-
.../pinot-batch-ingestion/v0_deprecated/pom.xml | 2 +-
pinot-plugins/pinot-file-system/pinot-adls/pom.xml | 2 +-
pinot-plugins/pinot-file-system/pinot-gcs/pom.xml | 2 +-
pinot-plugins/pinot-file-system/pinot-hdfs/pom.xml | 2 +-
pinot-plugins/pinot-file-system/pinot-s3/pom.xml | 2 +-
.../apache/pinot/plugin/filesystem/S3PinotFS.java | 89 +++--
pinot-plugins/pinot-file-system/pom.xml | 2 +-
.../pinot-input-format/pinot-avro-base/pom.xml | 2 +-
.../pinot-input-format/pinot-avro/pom.xml | 2 +-
.../pinot-confluent-avro/pom.xml | 2 +-
pinot-plugins/pinot-input-format/pinot-csv/pom.xml | 2 +-
.../pinot-input-format/pinot-json/pom.xml | 2 +-
pinot-plugins/pinot-input-format/pinot-orc/pom.xml | 2 +-
.../pinot-input-format/pinot-parquet/pom.xml | 2 +-
.../pinot-input-format/pinot-protobuf/pom.xml | 2 +-
.../pinot-input-format/pinot-thrift/pom.xml | 2 +-
pinot-plugins/pinot-input-format/pom.xml | 2 +-
.../pinot-stream-ingestion/pinot-kafka-0.9/pom.xml | 2 +-
.../pinot-stream-ingestion/pinot-kafka-2.0/pom.xml | 2 +-
.../pinot-kafka-base/pom.xml | 2 +-
pinot-plugins/pinot-stream-ingestion/pom.xml | 2 +-
pinot-plugins/pom.xml | 2 +-
pinot-server/pom.xml | 2 +-
pinot-spi/pom.xml | 2 +-
.../org/apache/pinot/spi/config/ConfigUtils.java | 95 +++++
.../apache/pinot/spi/config/table/FieldConfig.java | 1 +
.../pinot/spi/config/table/IndexingConfig.java | 5 +
.../SegmentsValidationAndRetentionConfig.java | 9 +
.../apache/pinot/spi/config/table/TableConfig.java | 1 +
...nsformConfig.java => BatchIngestionConfig.java} | 38 +-
.../table/{ => ingestion}/IngestionConfig.java | 30 +-
...ilterConfig.java => StreamIngestionConfig.java} | 19 +-
.../apache/pinot/spi/data/DateTimeFormatSpec.java | 58 +--
.../pinot/spi/ingestion/batch/BatchConfig.java | 133 +++++++
.../spi/ingestion/batch/BatchConfigProperties.java | 49 +++
.../pinot/spi/utils/IngestionConfigUtils.java | 95 +++++
.../java/org/apache/pinot/spi/utils/JsonUtils.java | 104 +++++-
.../org/apache/pinot/spi/utils/StringUtils.java | 4 +
.../spi/utils/builder/TableConfigBuilder.java | 2 +-
.../apache/pinot/spi/config/ConfigUtilsTest.java | 156 ++++++++
.../pinot/spi/ingestion/batch/BatchConfigTest.java | 171 +++++++++
.../pinot/spi/utils/IngestionConfigUtilsTest.java | 149 ++++++++
pinot-tools/pom.xml | 2 +-
.../pinot/tools/admin/command/JSONQuickstart.java | 5 +-
.../OfflineSegmentIntervalCheckerCommand.java | 8 +-
.../segment/converter/SegmentMergeCommand.java | 5 +-
.../batch/personJson/ingestionJobSpec.yaml | 139 -------
.../personJson_offline_table_config.json | 24 --
.../batch/personJson/personJson_schema.json | 9 -
.../batch/personJson/rawdata/personJson_data.csv | 101 -----
.../batch/personJson/sparkIngestionJobSpec.yaml | 147 --------
pom.xml | 2 +-
thirdeye/pom.xml | 21 +-
thirdeye/thirdeye-dashboard/pom.xml | 44 ---
.../thirdeye/detection/DetectionResource.java | 27 ++
.../performance/PerformanceMetricsTest.java | 215 +++++++++++
thirdeye/thirdeye-frontend/.eslintrc.js | 44 ++-
.../app/pods/components/alert-details/template.hbs | 27 +-
.../pods/components/detection-health/component.js | 35 +-
.../pods/components/detection-health/template.hbs | 154 ++++----
.../self-serve-alert-yaml-details/template.hbs | 148 +++-----
.../app/pods/components/stats-cards/template.hbs | 116 ++----
.../pods/components/timeseries-chart/component.js | 205 ++++++----
.../app/pods/home/index/template.hbs | 127 ++++---
.../app/pods/manage/explore/template.hbs | 25 +-
.../app/styles/components/button.scss | 34 +-
.../app/styles/shared/_styles.scss | 40 +-
.../share-custom-template/component-test.js | 34 +-
.../thirdeye-frontend/tests/utils/constants.js | 4 +-
thirdeye/thirdeye-pinot/pom.xml | 38 +-
.../registry/DetectionAlertRegistry.java | 50 +--
.../annotation/registry/DetectionRegistry.java | 25 +-
thirdeye/thirdeye-spi/pom.xml | 5 -
.../detection/performance/PerformanceMetric.java | 25 ++
.../performance/PerformanceMetricType.java | 5 +
.../detection/performance/PerformanceMetrics.java | 160 ++++++++
website/src/pages/download.js | 44 +++
website/src/pages/index.js | 6 +-
202 files changed, 4535 insertions(+), 2333 deletions(-)
create mode 100644 pinot-common/src/main/java/org/apache/pinot/common/function/scalar/ArrayFunctions.java
create mode 100644 pinot-core/src/main/java/org/apache/pinot/core/data/table/UnboundedConcurrentIndexedTable.java
copy pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/{EqualsTransformFunction.java => AndOperatorTransformFunction.java} (66%)
create mode 100644 pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/LogicalOperatorTransformFunction.java
copy pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/{EqualsTransformFunction.java => OrOperatorTransformFunction.java} (66%)
copy pinot-spi/src/main/java/org/apache/pinot/spi/utils/retry/NoDelayRetryPolicy.java => pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/AndOperatorTransformFunctionTest.java (69%)
create mode 100644 pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/LogicalOperatorTransformFunctionTest.java
copy pinot-spi/src/main/java/org/apache/pinot/spi/utils/retry/NoDelayRetryPolicy.java => pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/OrOperatorTransformFunctionTest.java (70%)
create mode 100644 pinot-spi/src/main/java/org/apache/pinot/spi/config/ConfigUtils.java
copy pinot-spi/src/main/java/org/apache/pinot/spi/config/table/ingestion/{TransformConfig.java => BatchIngestionConfig.java} (51%)
rename pinot-spi/src/main/java/org/apache/pinot/spi/config/table/{ => ingestion}/IngestionConfig.java (63%)
copy pinot-spi/src/main/java/org/apache/pinot/spi/config/table/ingestion/{FilterConfig.java => StreamIngestionConfig.java} (67%)
create mode 100644 pinot-spi/src/main/java/org/apache/pinot/spi/ingestion/batch/BatchConfig.java
create mode 100644 pinot-spi/src/main/java/org/apache/pinot/spi/ingestion/batch/BatchConfigProperties.java
create mode 100644 pinot-spi/src/main/java/org/apache/pinot/spi/utils/IngestionConfigUtils.java
create mode 100644 pinot-spi/src/test/java/org/apache/pinot/spi/config/ConfigUtilsTest.java
create mode 100644 pinot-spi/src/test/java/org/apache/pinot/spi/ingestion/batch/BatchConfigTest.java
create mode 100644 pinot-spi/src/test/java/org/apache/pinot/spi/utils/IngestionConfigUtilsTest.java
delete mode 100644 pinot-tools/src/main/resources/examples/batch/personJson/ingestionJobSpec.yaml
delete mode 100644 pinot-tools/src/main/resources/examples/batch/personJson/personJson_offline_table_config.json
delete mode 100644 pinot-tools/src/main/resources/examples/batch/personJson/personJson_schema.json
delete mode 100644 pinot-tools/src/main/resources/examples/batch/personJson/rawdata/personJson_data.csv
delete mode 100644 pinot-tools/src/main/resources/examples/batch/personJson/sparkIngestionJobSpec.yaml
create mode 100644 thirdeye/thirdeye-dashboard/src/test/java/org/apache/pinot/thirdeye/detection/performance/PerformanceMetricsTest.java
create mode 100644 thirdeye/thirdeye-spi/src/main/java/org/apache/pinot/thirdeye/detection/performance/PerformanceMetric.java
create mode 100644 thirdeye/thirdeye-spi/src/main/java/org/apache/pinot/thirdeye/detection/performance/PerformanceMetricType.java
create mode 100644 thirdeye/thirdeye-spi/src/main/java/org/apache/pinot/thirdeye/detection/performance/PerformanceMetrics.java
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org