You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by ja...@apache.org on 2019/09/24 01:16:31 UTC
[incubator-pinot] branch aggregation_function updated (6088ad0 ->
400bb61)
This is an automated email from the ASF dual-hosted git repository.
jackie pushed a change to branch aggregation_function
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git.
discard 6088ad0 Misc fixes for aggregation functions
add a1c5af7 [TE] Return the anomaly details to render snapshot (#4576)
add 05609e4 Use Apache Pinot License (#4584)
add 76e0c62 Adding Predicate supports for Bytes Column without dictionary (#4586)
add 68092ab Adding integration test for Hadoop pre-process job (#4577)
add 3c2f0b3 Adding example for kubernetes deployment on GKE (#4581)
add dbcbd2e [TE] Empty Entity report bug fix (#4598)
add 5e525d0 [TE] Exclude template/blacklist emails before sending out alert (#4601)
add 0a32309 [TE] fix create alert error message NPE (#4594)
add e70a781 Add ApacheCon Slide to README file (#4604)
add 67ea9bb [TE] Merger fix to consider anomalies generated earlier (#4565)
add f5017be [TE] Redesign to support multiple Alert Content formatters (#4599)
add add0f07 [TE] Update AI Availability table (#4606)
add fe7fff6 [TE] Add anomaly type for merged anomaly result (#4605)
add 8f9bb5d [TE] Remove the logic of reading stderr from phantomjs to avoid zombie processes (#4611)
add 45205e8 [TE] frontend - harleyjj/alert-details - handle reported anomaly for anomaly table (#4587)
add fe29287 [TE] frontend - harleyjj/auth - handle expired token and 401 errors plus redirect after login (#4580)
add 76fcc86 [TE] frontend - harleyjj/ai - fix small bug in AI Availability (#4600)
add 09207f9 [TE] consider anomaly type during merge (#4613)
add 8d593f3 [TE] Bug fix to initialize the configDAO (#4616)
add c7b647a [TE][Composite-Alert] Entity Anomaly Merger (#4609)
add c87b110 [TE] Add data set display name and show it on UI (#4619)
add 239237e [TE] the last time stamp filter out the anomalies (#4621)
add 44d1ce9 Remove import of sun.misc.Cleaner to make code compatible with jdk11 (#4624)
add 04c4dc2 Revert "Remove import of sun.misc.Cleaner to make code compatible with jdk11 (#4624)" (#4625)
add eda4176 [TE] [Notification] Refactor email filters to a generalized alert filter (#4620)
add 3668c79 Documentation update for realtime Pinot (#4630)
add f8c2cf3 Put order-by expressions in front of selection expreesions in selection DataSchema (#4629)
add 400bb61 Misc fixes for aggregation functions
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (6088ad0)
\
N -- N -- N refs/heads/aggregation_function (400bb61)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
No new revisions were added by this update.
Summary of changes:
README.md | 9 +-
docs/architecture.rst | 66 +++-
docs/code_modules.rst | 5 +-
docs/contribution_guidelines.rst | 3 +-
docs/pluggable_streams.rst | 9 +-
docs/pql_examples.rst | 9 +-
docs/tableconfig_schema.rst | 168 ++++++++-
docs/tuning_pinot.rst | 1 +
docs/tuning_realtime_performance.rst | 77 ++++
kubernetes/README.md | 27 ++
kubernetes/examples/gke/skaffold/README.md | 119 ++++++
kubernetes/examples/gke/skaffold/cleanup.sh | 39 ++
.../gke/skaffold/gke-storageclass-kafka-pd.yml | 28 ++
.../gke-storageclass-pinot-controller-pd.yml | 28 ++
.../skaffold/gke-storageclass-pinot-server-pd.yml | 28 ++
.../gke/skaffold/gke-storageclass-zk-pd.yml | 28 ++
kubernetes/examples/gke/skaffold/kafka.yml | 408 +++++++++++++++++++++
kubernetes/examples/gke/skaffold/pinot-broker.yml | 88 +++++
.../examples/gke/skaffold/pinot-controller.yml | 106 ++++++
.../examples/gke/skaffold/pinot-example-loader.yml | 43 +++
kubernetes/examples/gke/skaffold/pinot-server.yml | 101 +++++
.../examples/gke/skaffold/query-pinot-data.sh | 28 ++
kubernetes/examples/gke/skaffold/setup.sh | 51 +++
kubernetes/examples/gke/skaffold/skaffold.yaml | 35 ++
kubernetes/examples/gke/skaffold/zookeeper.yml | 80 ++++
.../common/function/AggregationFunctionType.java | 55 ++-
.../apache/pinot/common/query/ReduceService.java | 4 +-
.../org/apache/pinot/common/utils/DataSchema.java | 25 +-
.../org/apache/pinot/common/utils/DataTable.java | 12 +-
.../parsers/PinotQuery2BrokerRequestConverter.java | 7 +-
.../apache/pinot/pql/parsers/Pql2AstListener.java | 2 +-
.../pinot/pql/parsers/pql2/ast/OrderByAstNode.java | 57 +--
.../apache/pinot/pql/parsers/Pql2CompilerTest.java | 30 +-
.../request/BrokerRequestSerializationTest.java | 9 +-
.../core/common/datatable/DataTableImplV2.java | 21 +-
.../dociditerators/SVScanDocIdIterator.java | 11 +
.../operator/filter/ExpressionFilterOperator.java | 21 +-
.../BaseDictionaryBasedPredicateEvaluator.java | 10 +
.../BaseRawValueBasedPredicateEvaluator.java | 26 ++
.../predicate/EqualsPredicateEvaluatorFactory.java | 22 ++
.../predicate/InPredicateEvaluatorFactory.java | 26 ++
.../NotEqualsPredicateEvaluatorFactory.java | 22 ++
.../predicate/NotInPredicateEvaluatorFactory.java | 27 ++
.../filter/predicate/PredicateEvaluator.java | 17 +
.../predicate/RangePredicateEvaluatorFactory.java | 51 +++
.../operator/query/EmptySelectionOperator.java | 26 +-
.../core/operator/query/SelectionOnlyOperator.java | 126 +++++++
.../core/operator/query/SelectionOperator.java | 282 --------------
.../operator/query/SelectionOrderByOperator.java | 179 +++++++++
.../core/operator/transform/TransformOperator.java | 21 +-
.../apache/pinot/core/plan/SelectionPlanNode.java | 33 +-
.../apache/pinot/core/plan/TransformPlanNode.java | 64 ++--
.../function/AggregationFunctionFactory.java | 4 +-
.../core/query/reduce/BrokerReduceService.java | 262 ++++++++++---
.../query/selection/SelectionOperatorService.java | 251 +++----------
.../query/selection/SelectionOperatorUtils.java | 368 +++++++------------
.../name/NormalizedDateSegmentNameGenerator.java | 4 -
.../startree/plan/StarTreeTransformPlanNode.java | 3 +-
.../NoDictionaryEqualsPredicateEvaluatorsTest.java | 37 ++
.../NoDictionaryInPredicateEvaluatorTest.java | 40 ++
.../NoDictionaryRangePredicateEvaluatorTest.java | 57 +++
.../predicate/PredicateEvaluatorTestUtils.java | 7 +
...InnerSegmentSelectionMultiValueQueriesTest.java | 48 ++-
...nnerSegmentSelectionSingleValueQueriesTest.java | 60 ++-
.../DefaultAggregationExecutorTest.java | 2 +-
.../selection/SelectionOperatorServiceTest.java | 209 +++++++----
.../pinot/hadoop/job/SegmentCreationJob.java | 17 +-
.../pinot/hadoop/job/SegmentPreprocessingJob.java | 14 +-
.../job/mappers/SegmentPreprocessingMapper.java | 17 +-
.../pinot/hadoop/utils/JobPreparationHelper.java | 11 +
.../tests/ClusterIntegrationTestUtils.java | 19 +-
.../pinot/integration/tests/ClusterTest.java | 8 +-
...mentBuildPushOfflineClusterIntegrationTest.java | 97 ++---
.../app/mirage/factories/alert.js | 6 +-
.../app/pods/aiavailability/route.js | 30 +-
.../app/pods/application/route.js | 1 +
.../app/pods/components/alert-details/component.js | 2 +-
.../thirdeye-frontend/app/pods/home/index/route.js | 13 +
thirdeye/thirdeye-frontend/app/pods/login/route.js | 25 +-
.../app/pods/manage/alerts/index/route.js | 22 +-
.../app/pods/manage/alerts/route.js | 13 +-
.../app/pods/manage/explore/route.js | 39 +-
.../thirdeye-frontend/app/pods/manage/route.js | 13 +-
.../app/pods/manage/yaml/route.js | 26 +-
.../thirdeye-frontend/app/pods/rootcause/route.js | 13 +-
.../app/pods/services/query-cache/service.js | 8 +-
thirdeye/thirdeye-frontend/app/utils/anomaly.js | 8 +
.../thirdeye-frontend/app/utils/api/anomaly.js | 2 +-
.../acceptance/self-serve-alert-tuning-test.js | 2 -
.../thirdeye/alert/commons/AnomalyFeedConfig.java | 6 +-
.../alert/content/EmailContentFormatter.java | 62 ----
.../EmailContentFormatterConfiguration.java | 128 -------
.../thirdeye/alert/fetcher/BaseAnomalyFetcher.java | 4 +-
.../AnomalyType.java} | 27 +-
.../thirdeye/anomaly/alert/AlertJobContext.java | 1 +
.../thirdeye/anomaly/alert/AlertTaskInfo.java | 1 +
...nshotHelper.java => AlertScreenshotHelper.java} | 56 +--
.../anomaly/alert/util/AnomalyReportGenerator.java | 2 +-
.../thirdeye/anomaly/alert/util/EmailHelper.java | 29 +-
.../anomaly/alert/v2/AlertTaskRunnerV2.java | 40 +-
.../tasks/FunctionCreationOnboardingTask.java | 7 +-
.../onboard/tasks/NotificationOnboardingTask.java | 34 +-
.../AbstractModularizedAnomalyFunction.java | 2 +-
.../dashboard/resources/v2/DataResource.java | 13 +-
.../v2/rootcause/MetricEntityFormatter.java | 4 +-
.../thirdeye/datalayer/dto/AnomalyFunctionDTO.java | 7 +-
.../thirdeye/datalayer/dto/DatasetConfigDTO.java | 9 +
.../datalayer/entity/DatasetConfigIndex.java | 10 +
.../thirdeye/datalayer/pojo/DatasetConfigBean.java | 57 +--
.../datalayer/pojo/MergedAnomalyResultBean.java | 14 +
.../{StringUtils.java => ThirdEyeStringUtils.java} | 2 +-
.../pinot/thirdeye/detection/ConfigUtils.java | 29 ++
.../pinot/thirdeye/detection/DataProvider.java | 5 +
.../thirdeye/detection/DefaultDataProvider.java | 39 +-
.../alert/DetectionAlertFilterNotification.java | 62 ++++
.../alert/DetectionAlertFilterResult.java | 16 +-
.../alert/StatefulDetectionAlertFilter.java | 40 ++
.../filter/DimensionDetectionAlertFilter.java | 25 +-
.../alert/filter/PerUserDimensionAlertFilter.java | 19 +-
.../ToAllRecipientsDetectionAlertFilter.java | 36 +-
.../alert/scheme/DetectionEmailAlerter.java | 123 ++++---
.../thirdeye/detection/algorithm/MergeWrapper.java | 126 +++++--
.../thirdeye/detection/spi/model/AnomalySlice.java | 52 ++-
.../validators/DetectionConfigValidator.java | 20 +-
.../detection/wrapper/AnomalyDetectorWrapper.java | 4 +-
.../wrapper/BaselineFillingMergeWrapper.java | 19 +-
.../wrapper/ChildKeepingMergeWrapper.java | 5 +-
.../wrapper/EntityAnomalyMergeWrapper.java | 77 ++++
.../thirdeye/detection/wrapper/GrouperWrapper.java | 2 +-
.../thirdeye/detection/yaml/YamlResource.java | 30 +-
.../yaml/translator/DetectionConfigTranslator.java | 23 +-
.../translator/DetectionMetricAttributeHolder.java | 13 +-
.../formatter/DetectionConfigFormatter.java | 109 ++++--
.../commons/EmailEntity.java | 2 +-
.../content/BaseNotificationContent.java} | 327 +++++------------
.../notification/content/NotificationContent.java | 55 +++
.../content/templates/EntityGroupKeyContent.java} | 42 ++-
.../templates/HierarchicalAnomaliesContent.java} | 90 +++--
.../content/templates/MetricAnomaliesContent.java} | 48 +--
.../templates/OnboardingNotificationContent.java} | 42 ++-
.../formatter/ADContentFormatterContext.java} | 8 +-
.../formatter/AlertContentFormatterFactory.java} | 14 +-
.../formatter/channels/EmailContentFormatter.java | 125 +++++++
.../src/main/resources/schema/create-schema.sql | 2 +
.../commons/TestEmailContentFormatterFactory.java | 39 --
.../alert/fetcher/TestContinuumAnomalyFetcher.java | 7 +-
.../transform/testAnomalyRemovalFunction.java | 2 +-
...UtilsTest.java => ThirdEyeStringUtilsTest.java} | 4 +-
.../pinot/thirdeye/detection/DataProviderTest.java | 2 +-
.../detection/alert/filter/AlertFilterUtils.java | 65 ++++
.../filter/DimensionDetectionAlertFilterTest.java | 33 +-
.../filter/PerUserDimensionAlertFilterTest.java | 38 +-
.../ToAllRecipientsDetectionAlertFilterTest.java | 43 ++-
.../DetectionTimeWindowSuppressorTest.java | 14 +-
.../detection/algorithm/MergeWrapperTest.java | 58 ++-
.../wrapper/EntityAnomalyMergeWrapperTest.java | 136 +++++++
.../formatter/DetectionConfigFormatterTest.java | 14 +-
.../ContentFormatterUtils.java | 8 +-
.../templates/TestEntityGroupKeyContent.java} | 31 +-
.../TestHierarchicalAnomaliesContent.java} | 20 +-
.../templates/TestMetricAnomaliesContent.java} | 19 +-
.../TestOnboardingNotificationContent.java} | 19 +-
.../TestAlertContentFormatterFactory.java | 41 +++
.../compositePipelineTranslatorTestResult-1.json | 93 ++---
164 files changed, 4939 insertions(+), 2419 deletions(-)
create mode 100644 docs/tuning_realtime_performance.rst
create mode 100644 kubernetes/README.md
create mode 100644 kubernetes/examples/gke/skaffold/README.md
create mode 100755 kubernetes/examples/gke/skaffold/cleanup.sh
create mode 100644 kubernetes/examples/gke/skaffold/gke-storageclass-kafka-pd.yml
create mode 100644 kubernetes/examples/gke/skaffold/gke-storageclass-pinot-controller-pd.yml
create mode 100644 kubernetes/examples/gke/skaffold/gke-storageclass-pinot-server-pd.yml
create mode 100644 kubernetes/examples/gke/skaffold/gke-storageclass-zk-pd.yml
create mode 100644 kubernetes/examples/gke/skaffold/kafka.yml
create mode 100644 kubernetes/examples/gke/skaffold/pinot-broker.yml
create mode 100644 kubernetes/examples/gke/skaffold/pinot-controller.yml
create mode 100644 kubernetes/examples/gke/skaffold/pinot-example-loader.yml
create mode 100644 kubernetes/examples/gke/skaffold/pinot-server.yml
create mode 100755 kubernetes/examples/gke/skaffold/query-pinot-data.sh
create mode 100755 kubernetes/examples/gke/skaffold/setup.sh
create mode 100644 kubernetes/examples/gke/skaffold/skaffold.yaml
create mode 100644 kubernetes/examples/gke/skaffold/zookeeper.yml
create mode 100644 pinot-core/src/main/java/org/apache/pinot/core/operator/query/SelectionOnlyOperator.java
delete mode 100644 pinot-core/src/main/java/org/apache/pinot/core/operator/query/SelectionOperator.java
create mode 100644 pinot-core/src/main/java/org/apache/pinot/core/operator/query/SelectionOrderByOperator.java
delete mode 100644 thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/alert/content/EmailContentFormatter.java
delete mode 100644 thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/alert/content/EmailContentFormatterConfiguration.java
copy thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/{alert/commons/EmailContentFormatterFactory.java => anomaly/AnomalyType.java} (55%)
rename thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/anomaly/alert/util/{EmailScreenshotHelper.java => AlertScreenshotHelper.java} (56%)
rename thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datalayer/util/{StringUtils.java => ThirdEyeStringUtils.java} (98%)
create mode 100644 thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/alert/DetectionAlertFilterNotification.java
create mode 100644 thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/wrapper/EntityAnomalyMergeWrapper.java
rename thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/{alert => notification}/commons/EmailEntity.java (97%)
rename thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/{alert/content/BaseEmailContentFormatter.java => notification/content/BaseNotificationContent.java} (70%)
create mode 100644 thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/notification/content/NotificationContent.java
rename thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/{alert/content/EntityGroupKeyContentFormatter.java => notification/content/templates/EntityGroupKeyContent.java} (84%)
rename thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/{alert/content/HierarchicalAnomaliesEmailContentFormatter.java => notification/content/templates/HierarchicalAnomaliesContent.java} (67%)
rename thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/{alert/content/MetricAnomaliesEmailContentFormatter.java => notification/content/templates/MetricAnomaliesContent.java} (81%)
rename thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/{alert/content/OnboardingNotificationEmailContentFormatter.java => notification/content/templates/OnboardingNotificationContent.java} (75%)
rename thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/{alert/content/EmailContentFormatterContext.java => notification/formatter/ADContentFormatterContext.java} (88%)
rename thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/{alert/commons/EmailContentFormatterFactory.java => notification/formatter/AlertContentFormatterFactory.java} (65%)
create mode 100644 thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/notification/formatter/channels/EmailContentFormatter.java
delete mode 100644 thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/alert/commons/TestEmailContentFormatterFactory.java
rename thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/datalayer/util/{StringUtilsTest.java => ThirdEyeStringUtilsTest.java} (75%)
create mode 100644 thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/alert/filter/AlertFilterUtils.java
create mode 100644 thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/wrapper/EntityAnomalyMergeWrapperTest.java
rename thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/{alert/content => notification}/ContentFormatterUtils.java (83%)
rename thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/{alert/content/TestEntityGroupKeyEmailContentFormatter.java => notification/content/templates/TestEntityGroupKeyContent.java} (91%)
rename thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/{alert/content/TestHierarchicalAnomaliesEmailContentFormatter.java => notification/content/templates/TestHierarchicalAnomaliesContent.java} (89%)
rename thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/{alert/content/TestMetricAnomaliesEmailContentFormatter.java => notification/content/templates/TestMetricAnomaliesContent.java} (88%)
rename thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/{alert/content/TestOnboardingNotificationContentFormatter.java => notification/content/templates/TestOnboardingNotificationContent.java} (89%)
create mode 100644 thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/notification/formatter/TestAlertContentFormatterFactory.java
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org