You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by xi...@apache.org on 2019/11/24 22:37:53 UTC
[incubator-pinot] branch pinot-ingestion-refactor updated (6b59574
-> a77676e)
This is an automated email from the ASF dual-hosted git repository.
xiangfu pushed a change to branch pinot-ingestion-refactor
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git.
discard 6b59574 Adding support for retry and back-off
discard 7df1594 update with new API
discard c7dfc90 Adding config to control spark job push parallism
discard 66f043e Fixing parent directory creation issue
discard bb5b808 don't use default conf
discard 7b476aa fixing class: com.databricks.backend.daemon.data.client.DBFS not serializable issue
discard 37d6e55 Ensure segment push job is Serializable
discard 5cde44b Make spark job an option to parallel push segments
discard 80d2d79 Address comments
discard 9636308 Move segment tars recursively and override existed files
discard f9c983c Initial refactor
discard becc214 Make temp directory with uuid appended
discard b56cc1c Make PushLocation Serializable
discard c05d660 Use SparkContext.getOrCreate() to use shared SparkContext if possible.
discard c272587 Initial commit for pinot-spark
add 81703be [TE] Centralized cache proof-of-concept for anomaly detection via Couchbase (#4761)
add c720b99 [TE] quick fix for error in retriving legacy anomaly (#4843)
add 5e5b32f Remove the support of Hocon based config (#4841)
add 9403cf1 Remove the redundant testSqlQueriesFromQueryFile() in integration tests (#4848)
add e1b3512 Prepare for pinot release 0.3.0 (#4849)
add e6e7bf9 [TE] Use default centralized cache settings if exception occurs during init (#4845)
add 1cc296b [TE] Fix duplicate entries in entity report (#4840)
add 759eee9 Add getFinalResultColumnType to be used in constructing ResultTable::dataSchema (#4852)
add 1bea1c2 Initial commit for pinot-spark
add d678005 Use SparkContext.getOrCreate() to use shared SparkContext if possible.
add bf53e3b Make PushLocation Serializable
add 3fc0caf Make temp directory with uuid appended
add 0f02a45 Initial refactor
add 1fe80e9 Move segment tars recursively and override existed files
add 0f420cf Address comments
add 79811cb Make spark job an option to parallel push segments
add 6f51b3c Ensure segment push job is Serializable
add 51f7280 fixing class: com.databricks.backend.daemon.data.client.DBFS not serializable issue
add 1211b99 don't use default conf
add e352860 Fixing parent directory creation issue
add ae6ed97 Adding config to control spark job push parallism
add b3c6836 update with new API
add a77676e Adding support for retry and back-off
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (6b59574)
\
N -- N -- N refs/heads/pinot-ingestion-refactor (a77676e)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
No new revisions were added by this update.
Summary of changes:
pinot-api/pom.xml | 2 +-
pinot-azure-filesystem/pom.xml | 2 +-
pinot-broker/pom.xml | 2 +-
pinot-common/pom.xml | 10 +-
.../config/AdjustTableNameChildKeyTransformer.java | 70 ----
.../pinot/common/config/ChildKeyHandler.java | 45 ---
.../pinot/common/config/ChildKeyTransformer.java | 33 --
.../pinot/common/config/ColumnPartitionConfig.java | 16 +-
.../config/ColumnPartitionMapChildKeyHandler.java | 28 --
.../apache/pinot/common/config/CombinedConfig.java | 110 -----
.../pinot/common/config/CombinedConfigLoader.java | 171 --------
...CombinedConfigSeparatorChildKeyTransformer.java | 132 ------
.../pinot/common/config/CompletionConfig.java | 4 +-
.../org/apache/pinot/common/config/ConfigDoc.java | 37 --
.../org/apache/pinot/common/config/ConfigKey.java | 31 --
.../common/config/ConfigNodeLifecycleAware.java | 34 --
.../apache/pinot/common/config/Deserializer.java | 445 ---------------------
.../org/apache/pinot/common/config/Duration.java | 77 ----
.../apache/pinot/common/config/DurationDsl.java | 57 ---
.../apache/pinot/common/config/IndexingConfig.java | 40 +-
.../common/config/NamedListChildKeyHandler.java | 75 ----
.../apache/pinot/common/config/NestedConfig.java | 31 --
.../pinot/common/config/PinotTaskConfig.java | 11 +-
.../apache/pinot/common/config/QuotaConfig.java | 5 +-
.../config/RemapTableTypesChildKeyTransformer.java | 64 ---
.../common/config/ReplicaGroupStrategyConfig.java | 6 -
.../apache/pinot/common/config/RoutingConfig.java | 2 -
.../common/config/SegmentPartitionConfig.java | 14 +-
.../SegmentsValidationAndRetentionConfig.java | 26 --
.../org/apache/pinot/common/config/Serializer.java | 197 ---------
.../common/config/SimpleMapChildKeyHandler.java | 50 ---
.../apache/pinot/common/config/SingleKeyDsl.java | 30 --
.../pinot/common/config/StarTreeIndexConfig.java | 7 -
.../apache/pinot/common/config/TableConfig.java | 26 +-
.../pinot/common/config/TableCustomConfig.java | 2 -
.../pinot/common/config/TableTaskConfig.java | 3 -
.../pinot/common/config/TagOverrideConfig.java | 31 +-
.../config/TaskConfigMapChildKeyHandler.java | 59 ---
.../org/apache/pinot/common/config/Tenant.java | 21 +-
.../apache/pinot/common/config/TenantConfig.java | 34 +-
.../common/config/TypedMapChildKeyHandler.java | 74 ----
.../pinot/common/config/UseChildKeyHandler.java | 31 --
.../common/config/UseChildKeyTransformers.java | 34 --
.../org/apache/pinot/common/config/UseDsl.java | 35 --
.../config/instance/InstanceAssignmentConfig.java | 16 +-
...InstanceAssignmentConfigMapChildKeyHandler.java | 28 --
.../config/instance/InstanceConstraintConfig.java | 6 +-
.../InstanceReplicaGroupPartitionConfig.java | 21 +-
.../config/instance/InstanceTagPoolConfig.java | 15 +-
.../pinot/common/data/DateTimeFieldSpec.java | 16 +-
.../data/DateTimeFieldSpecChildKeyHandler.java | 31 --
.../data/DimensionFieldSpecChildKeyHandler.java | 31 --
.../org/apache/pinot/common/data/FieldSpec.java | 24 +-
.../apache/pinot/common/data/MetricFieldSpec.java | 25 +-
.../data/MetricFieldSpecChildKeyHandler.java | 31 --
.../java/org/apache/pinot/common/data/Schema.java | 141 +++----
.../pinot/common/data/StarTreeIndexSpec.java | 7 -
.../apache/pinot/common/data/TimeFieldSpec.java | 69 ++--
.../pinot/common/data/TimeGranularitySpec.java | 34 +-
.../org/apache/pinot/startree/hll/HllConfig.java | 82 ++--
.../common/config/TestConfigSerialization.java | 70 ----
.../src/test/resources/test-table-config.conf | 32 --
pinot-connectors/pinot-connector-kafka-0.9/pom.xml | 2 +-
pinot-connectors/pinot-connector-kafka-2.0/pom.xml | 4 +-
.../pinot-connector-kafka-base/pom.xml | 2 +-
pinot-connectors/pom.xml | 4 +-
pinot-controller/pom.xml | 2 +-
.../resources/PinotTableConfigRestletResource.java | 195 ---------
.../helix/ControllerRequestURLBuilder.java | 8 -
pinot-core/pom.xml | 2 +-
.../aggregation/function/AggregationFunction.java | 6 +
.../function/AvgAggregationFunction.java | 5 +
.../function/CountAggregationFunction.java | 5 +
.../function/DistinctAggregationFunction.java | 5 +
.../function/DistinctCountAggregationFunction.java | 5 +
.../DistinctCountHLLAggregationFunction.java | 5 +
.../DistinctCountRawHLLAggregationFunction.java | 5 +
.../function/FastHLLAggregationFunction.java | 5 +
.../function/MaxAggregationFunction.java | 5 +
.../function/MinAggregationFunction.java | 5 +
.../function/MinMaxRangeAggregationFunction.java | 5 +
.../function/PercentileAggregationFunction.java | 5 +
.../function/PercentileEstAggregationFunction.java | 5 +
.../PercentileTDigestAggregationFunction.java | 5 +
.../function/SumAggregationFunction.java | 5 +
pinot-distribution/pom.xml | 2 +-
pinot-hadoop-filesystem/pom.xml | 2 +-
pinot-ingestion-jobs/pinot-hadoop/pom.xml | 2 +-
pinot-integration-tests/pom.xml | 2 +-
.../tests/BaseClusterIntegrationTest.java | 40 +-
.../tests/BaseClusterIntegrationTestSet.java | 23 +-
.../pinot/integration/tests/ClusterTest.java | 52 +--
.../ControllerPeriodicTasksIntegrationTests.java | 13 +-
...vertToRawIndexMinionClusterIntegrationTest.java | 5 -
.../DefaultCommitterRealtimeIntegrationTest.java | 5 -
...mentBuildPushOfflineClusterIntegrationTest.java | 7 -
.../tests/HybridClusterIntegrationTest.java | 14 -
.../MultiNodesOfflineClusterIntegrationTest.java | 7 -
.../tests/NewConfigApplyIntegrationTest.java | 116 ------
.../tests/OfflineClusterIntegrationTest.java | 20 -
.../tests/RealtimeClusterIntegrationTest.java | 15 -
.../src/test/resources/mytable-updated.conf | 16 -
.../src/test/resources/mytable.conf | 16 -
.../src/test/resources/profiles/test1.conf | 1 -
.../src/test/resources/profiles/test2.conf | 1 -
pinot-minion/pom.xml | 2 +-
pinot-orc/pom.xml | 2 +-
pinot-parquet/pom.xml | 2 +-
pinot-perf/pom.xml | 2 +-
pinot-server/pom.xml | 2 +-
pinot-tools/pom.xml | 2 +-
.../pinot/tools/admin/PinotAdministrator.java | 2 -
.../admin/command/ApplyTableConfigCommand.java | 150 -------
.../manager/collector/CompressedFilePathIter.java | 58 +--
.../pinot/tools/tuner/strategy/FrequencyImpl.java | 40 +-
.../tools/tuner/strategy/ParserBasedImpl.java | 95 ++---
.../strategy/QuantileAnalysisAccumulator.java | 19 +-
.../tools/tuner/strategy/QuantileAnalysisImpl.java | 72 ++--
pinot-transport/pom.xml | 2 +-
pom.xml | 13 +-
.../config/data-sources/cache-config.yml | 20 +
thirdeye/thirdeye-pinot/pom.xml | 6 +
.../anomaly/utils/ThirdeyeMetricsUtil.java | 9 +
.../api/user/dashboard/UserDashboardResource.java | 11 +-
.../thirdeye/common/ThirdEyeConfiguration.java | 18 +-
.../dashboard/ThirdEyeDashboardApplication.java | 2 +-
.../dashboard/resources/v2/AnomaliesResource.java | 2 +-
.../datasource/RelationalThirdEyeResponse.java | 3 +
.../thirdeye/datasource/ThirdEyeCacheRegistry.java | 55 ++-
.../datasource/loader/DefaultTimeSeriesLoader.java | 20 +-
.../pinot/resultset/ThirdEyeResultSetUtils.java | 13 +-
.../datasource/sql/SqlResponseCacheLoader.java | 2 -
.../thirdeye/detection/DefaultDataProvider.java | 85 ++--
.../detection/DetectionPipelineTaskRunner.java | 2 +-
.../thirdeye/detection/DetectionResource.java | 2 +-
.../detection/alert/DetectionAlertTaskFactory.java | 2 +-
.../detection/algorithm/DimensionWrapper.java | 19 +
.../thirdeye/detection/cache/CacheConfig.java | 73 ++++
.../detection/cache/CacheConfigLoader.java | 35 +-
.../cache/CacheConstants.java} | 36 +-
.../thirdeye/detection/cache/CacheDataSource.java | 48 +++
.../detection/cache/CentralizedCacheConfig.java | 69 ++++
.../detection/cache/CouchbaseCacheDAO.java | 159 ++++++++
.../detection/cache/DefaultTimeSeriesCache.java | 197 +++++++++
.../detection/cache/ThirdEyeCacheRequest.java | 64 +++
.../detection/cache/ThirdEyeCacheResponse.java | 181 +++++++++
.../thirdeye/detection/cache/TimeSeriesCache.java | 49 +++
.../detection/cache/TimeSeriesDataPoint.java | 104 +++++
.../finetune/GridSearchTuningAlgorithm.java | 2 +-
.../onboard/YamlOnboardingTaskRunner.java | 2 +-
.../detection/wrapper/AnomalyDetectorWrapper.java | 38 +-
.../thirdeye/detection/yaml/YamlResource.java | 5 +-
.../content/templates/EntityGroupKeyContent.java | 8 +-
.../org/apache/pinot/thirdeye/util/CacheUtils.java | 89 +++++
.../apache/pinot/thirdeye/util/ThirdEyeUtils.java | 21 +
.../csv/CSVThirdEyeDataSourceIntegrationTest.java | 1 +
.../MockThirdEyeDataSourceIntegrationTest.java | 1 +
.../pinot/thirdeye/detection/DataProviderTest.java | 2 +-
.../detection/cache/ThirdEyeCacheResponseTest.java | 337 ++++++++++++++++
.../detection/cache/TimeSeriesCacheTest.java | 310 ++++++++++++++
.../integration/NotificationTaskSchedulerTest.java | 2 +-
.../templates/TestMetricAnomaliesContent.java | 2 +-
.../apache/pinot/thirdeye/util/CacheUtilTest.java | 104 +++++
.../test/resources/data-sources/cache-config.yml | 23 ++
164 files changed, 2573 insertions(+), 3670 deletions(-)
delete mode 100644 pinot-common/src/main/java/org/apache/pinot/common/config/AdjustTableNameChildKeyTransformer.java
delete mode 100644 pinot-common/src/main/java/org/apache/pinot/common/config/ChildKeyHandler.java
delete mode 100644 pinot-common/src/main/java/org/apache/pinot/common/config/ChildKeyTransformer.java
delete mode 100644 pinot-common/src/main/java/org/apache/pinot/common/config/ColumnPartitionMapChildKeyHandler.java
delete mode 100644 pinot-common/src/main/java/org/apache/pinot/common/config/CombinedConfig.java
delete mode 100644 pinot-common/src/main/java/org/apache/pinot/common/config/CombinedConfigLoader.java
delete mode 100644 pinot-common/src/main/java/org/apache/pinot/common/config/CombinedConfigSeparatorChildKeyTransformer.java
delete mode 100644 pinot-common/src/main/java/org/apache/pinot/common/config/ConfigDoc.java
delete mode 100644 pinot-common/src/main/java/org/apache/pinot/common/config/ConfigKey.java
delete mode 100644 pinot-common/src/main/java/org/apache/pinot/common/config/ConfigNodeLifecycleAware.java
delete mode 100644 pinot-common/src/main/java/org/apache/pinot/common/config/Deserializer.java
delete mode 100644 pinot-common/src/main/java/org/apache/pinot/common/config/Duration.java
delete mode 100644 pinot-common/src/main/java/org/apache/pinot/common/config/DurationDsl.java
delete mode 100644 pinot-common/src/main/java/org/apache/pinot/common/config/NamedListChildKeyHandler.java
delete mode 100644 pinot-common/src/main/java/org/apache/pinot/common/config/NestedConfig.java
delete mode 100644 pinot-common/src/main/java/org/apache/pinot/common/config/RemapTableTypesChildKeyTransformer.java
delete mode 100644 pinot-common/src/main/java/org/apache/pinot/common/config/Serializer.java
delete mode 100644 pinot-common/src/main/java/org/apache/pinot/common/config/SimpleMapChildKeyHandler.java
delete mode 100644 pinot-common/src/main/java/org/apache/pinot/common/config/SingleKeyDsl.java
delete mode 100644 pinot-common/src/main/java/org/apache/pinot/common/config/TaskConfigMapChildKeyHandler.java
delete mode 100644 pinot-common/src/main/java/org/apache/pinot/common/config/TypedMapChildKeyHandler.java
delete mode 100644 pinot-common/src/main/java/org/apache/pinot/common/config/UseChildKeyHandler.java
delete mode 100644 pinot-common/src/main/java/org/apache/pinot/common/config/UseChildKeyTransformers.java
delete mode 100644 pinot-common/src/main/java/org/apache/pinot/common/config/UseDsl.java
delete mode 100644 pinot-common/src/main/java/org/apache/pinot/common/config/instance/InstanceAssignmentConfigMapChildKeyHandler.java
delete mode 100644 pinot-common/src/main/java/org/apache/pinot/common/data/DateTimeFieldSpecChildKeyHandler.java
delete mode 100644 pinot-common/src/main/java/org/apache/pinot/common/data/DimensionFieldSpecChildKeyHandler.java
delete mode 100644 pinot-common/src/main/java/org/apache/pinot/common/data/MetricFieldSpecChildKeyHandler.java
delete mode 100644 pinot-common/src/test/java/org/apache/pinot/common/config/TestConfigSerialization.java
delete mode 100644 pinot-common/src/test/resources/test-table-config.conf
delete mode 100644 pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotTableConfigRestletResource.java
delete mode 100644 pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/NewConfigApplyIntegrationTest.java
delete mode 100644 pinot-integration-tests/src/test/resources/mytable-updated.conf
delete mode 100644 pinot-integration-tests/src/test/resources/mytable.conf
delete mode 100644 pinot-integration-tests/src/test/resources/profiles/test1.conf
delete mode 100644 pinot-integration-tests/src/test/resources/profiles/test2.conf
delete mode 100644 pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/ApplyTableConfigCommand.java
create mode 100644 thirdeye/thirdeye-pinot/config/data-sources/cache-config.yml
create mode 100644 thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/cache/CacheConfig.java
copy pinot-common/src/main/java/org/apache/pinot/common/restlet/resources/ResourceUtils.java => thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/cache/CacheConfigLoader.java (53%)
copy thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/{dataframe/util/RequestContainer.java => detection/cache/CacheConstants.java} (53%)
create mode 100644 thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/cache/CacheDataSource.java
create mode 100644 thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/cache/CentralizedCacheConfig.java
create mode 100644 thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/cache/CouchbaseCacheDAO.java
create mode 100644 thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/cache/DefaultTimeSeriesCache.java
create mode 100644 thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/cache/ThirdEyeCacheRequest.java
create mode 100644 thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/cache/ThirdEyeCacheResponse.java
create mode 100644 thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/cache/TimeSeriesCache.java
create mode 100644 thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/cache/TimeSeriesDataPoint.java
create mode 100644 thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/util/CacheUtils.java
create mode 100644 thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/cache/ThirdEyeCacheResponseTest.java
create mode 100644 thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/cache/TimeSeriesCacheTest.java
create mode 100644 thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/util/CacheUtilTest.java
create mode 100644 thirdeye/thirdeye-pinot/src/test/resources/data-sources/cache-config.yml
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org