You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by ap...@apache.org on 2020/03/06 23:52:54 UTC
[incubator-pinot] 05/05: database bootstrap with test application
This is an automated email from the ASF dual-hosted git repository.
apucher pushed a commit to branch docker-pinot-quickstart-thirdeye
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
commit 9200d0b2d4f8fd2e112dbb38203db01a5a7ab7ac
Author: Alexander Pucher <al...@alexpucher.com>
AuthorDate: Fri Mar 6 14:46:36 2020 -0800
database bootstrap with test application
---
docker/images/pinot-thirdeye/bin/start-thirdeye.sh | 7 +-
.../config/pinot-quickstart/bootstrap.sql | 4 +
.../batch/airlineStats/airlineStats_schema.json | 218 +++++++++++----------
.../stream/airlineStats/airlineStats_schema.json | 218 +++++++++++----------
4 files changed, 230 insertions(+), 217 deletions(-)
diff --git a/docker/images/pinot-thirdeye/bin/start-thirdeye.sh b/docker/images/pinot-thirdeye/bin/start-thirdeye.sh
index 08b30e7..f16169f 100755
--- a/docker/images/pinot-thirdeye/bin/start-thirdeye.sh
+++ b/docker/images/pinot-thirdeye/bin/start-thirdeye.sh
@@ -32,10 +32,15 @@ sleep 1
echo "Creating ThirdEye database schema"
java -cp "./bin/thirdeye-pinot-1.0-SNAPSHOT.jar" org.h2.tools.RunScript -user "sa" -password "sa" -url "jdbc:h2:tcp:localhost/h2db" -script "zip:./bin/thirdeye-pinot-1.0-SNAPSHOT.jar!/schema/create-schema.sql"
+if [ -f "${CONFIG_DIR}/bootstrap.sql" ]; then
+ echo "Running database bootstrap script ${CONFIG_DIR}/bootstrap.sql"
+ java -cp "./bin/thirdeye-pinot-1.0-SNAPSHOT.jar" org.h2.tools.RunScript -user "sa" -password "sa" -url "jdbc:h2:tcp:localhost/h2db" -script "${CONFIG_DIR}/bootstrap.sql"
+fi
+
echo "Running thirdeye backend config: ${CONFIG_DIR}"
[ -f "${CONFIG_DIR}/data-sources/data-sources-config-backend.yml" ] && cp "${CONFIG_DIR}/data-sources/data-sources-config-backend.yml" "${CONFIG_DIR}/data-sources/data-sources-config.yml"
java -cp "./bin/thirdeye-pinot-1.0-SNAPSHOT.jar" org.apache.pinot.thirdeye.anomaly.ThirdEyeAnomalyApplication "${CONFIG_DIR}" &
-sleep 5
+sleep 10
echo "Running thirdeye frontend config: ${CONFIG_DIR}"
[ -f "${CONFIG_DIR}/data-sources/data-sources-config-frontend.yml" ] && cp "${CONFIG_DIR}/data-sources/data-sources-config-frontend.yml" "${CONFIG_DIR}/data-sources/data-sources-config.yml"
diff --git a/docker/images/pinot-thirdeye/config/pinot-quickstart/bootstrap.sql b/docker/images/pinot-thirdeye/config/pinot-quickstart/bootstrap.sql
new file mode 100644
index 0000000..b9ac3d0
--- /dev/null
+++ b/docker/images/pinot-thirdeye/config/pinot-quickstart/bootstrap.sql
@@ -0,0 +1,4 @@
+INSERT INTO generic_json_entity (id, json_val, beanClass) VALUES (null, '{"id":null,"application":"test","recipients":""}', 'org.apache.pinot.thirdeye.datalayer.pojo.ApplicationBean');
+
+INSERT INTO application_index (base_id, application, recipients) VALUES (1, 'test', '');
+
diff --git a/pinot-tools/src/main/resources/examples/batch/airlineStats/airlineStats_schema.json b/pinot-tools/src/main/resources/examples/batch/airlineStats/airlineStats_schema.json
index 0c058c5..02143ca 100644
--- a/pinot-tools/src/main/resources/examples/batch/airlineStats/airlineStats_schema.json
+++ b/pinot-tools/src/main/resources/examples/batch/airlineStats/airlineStats_schema.json
@@ -1,14 +1,21 @@
{
- "metricFieldSpecs": [{
- "dataType": "INT",
- "name": "ActualElapsedTime"
- },
+ "metricFieldSpecs": [
+ ],
+ "dimensionFieldSpecs": [
+ {
+ "dataType": "INT",
+ "name": "ActualElapsedTime"
+ },
{
"dataType": "INT",
"name": "AirTime"
},
{
"dataType": "INT",
+ "name": "AirlineID"
+ },
+ {
+ "dataType": "INT",
"name": "ArrDel15"
},
{
@@ -24,6 +31,10 @@
"name": "ArrTime"
},
{
+ "dataType": "STRING",
+ "name": "ArrTimeBlk"
+ },
+ {
"dataType": "INT",
"name": "ArrivalDelayGroups"
},
@@ -40,11 +51,31 @@
"name": "CRSElapsedTime"
},
{
+ "dataType": "STRING",
+ "name": "CancellationCode"
+ },
+ {
+ "dataType": "INT",
+ "name": "Cancelled"
+ },
+ {
+ "dataType": "STRING",
+ "name": "Carrier"
+ },
+ {
"dataType": "INT",
"name": "CarrierDelay"
},
{
"dataType": "INT",
+ "name": "DayOfWeek"
+ },
+ {
+ "dataType": "INT",
+ "name": "DayofMonth"
+ },
+ {
+ "dataType": "INT",
"name": "DepDel15"
},
{
@@ -60,129 +91,152 @@
"name": "DepTime"
},
{
+ "dataType": "STRING",
+ "name": "DepTimeBlk"
+ },
+ {
"dataType": "INT",
"name": "DepartureDelayGroups"
},
{
- "dataType": "INT",
- "name": "Distance"
+ "dataType": "STRING",
+ "name": "Dest"
},
{
"dataType": "INT",
- "name": "DivActualElapsedTime"
+ "name": "DestAirportID"
},
{
"dataType": "INT",
- "name": "DivArrDelay"
+ "name": "DestAirportSeqID"
},
{
"dataType": "INT",
- "name": "DivDistance"
+ "name": "DestCityMarketID"
},
{
- "dataType": "INT",
- "name": "FirstDepTime"
+ "dataType": "STRING",
+ "name": "DestCityName"
+ },
+ {
+ "dataType": "STRING",
+ "name": "DestState"
},
{
"dataType": "INT",
- "name": "Flights"
+ "name": "DestStateFips"
+ },
+ {
+ "dataType": "STRING",
+ "name": "DestStateName"
},
{
"dataType": "INT",
- "name": "LateAircraftDelay"
+ "name": "DestWac"
},
{
"dataType": "INT",
- "name": "LongestAddGTime"
+ "name": "Distance"
},
{
"dataType": "INT",
- "name": "NASDelay"
+ "name": "DistanceGroup"
},
{
"dataType": "INT",
- "name": "SecurityDelay"
+ "name": "DivActualElapsedTime"
},
{
"dataType": "INT",
- "name": "TaxiIn"
+ "name": "DivAirportIDs",
+ "singleValueField": false
},
{
"dataType": "INT",
- "name": "TaxiOut"
+ "name": "DivAirportLandings"
},
{
"dataType": "INT",
- "name": "WheelsOn"
+ "name": "DivAirportSeqIDs",
+ "singleValueField": false
+ },
+ {
+ "dataType": "STRING",
+ "name": "DivAirports",
+ "singleValueField": false
},
{
"dataType": "INT",
- "name": "WheelsOff"
+ "name": "DivArrDelay"
},
{
"dataType": "INT",
- "name": "WeatherDelay"
+ "name": "DivDistance"
},
{
"dataType": "INT",
- "name": "TotalAddGTime"
- }
- ],
- "dimensionFieldSpecs": [{
- "dataType": "STRING",
- "name": "CancellationCode"
- },
+ "name": "DivLongestGTimes",
+ "singleValueField": false
+ },
{
"dataType": "INT",
- "name": "Cancelled"
+ "name": "DivReachedDest"
},
{
"dataType": "STRING",
- "name": "Carrier"
+ "name": "DivTailNums",
+ "singleValueField": false
},
{
"dataType": "INT",
- "name": "DayOfWeek"
+ "name": "DivTotalGTimes",
+ "singleValueField": false
},
{
"dataType": "INT",
- "name": "DayofMonth"
+ "name": "DivWheelsOffs",
+ "singleValueField": false
},
{
- "dataType": "STRING",
- "name": "Dest"
+ "dataType": "INT",
+ "name": "DivWheelsOns",
+ "singleValueField": false
},
{
"dataType": "INT",
- "name": "DestAirportID"
+ "name": "Diverted"
},
{
"dataType": "INT",
- "name": "DestAirportSeqID"
+ "name": "FirstDepTime"
+ },
+ {
+ "dataType": "STRING",
+ "name": "FlightDate"
},
{
"dataType": "INT",
- "name": "DestCityMarketID"
+ "name": "FlightNum"
},
{
- "dataType": "STRING",
- "name": "DestCityName"
+ "dataType": "INT",
+ "name": "Flights"
},
{
- "dataType": "STRING",
- "name": "DestState"
+ "dataType": "INT",
+ "name": "LateAircraftDelay"
},
{
"dataType": "INT",
- "name": "DestStateFips"
+ "name": "LongestAddGTime"
},
{
- "dataType": "STRING",
- "name": "DestStateName"
+ "dataType": "INT",
+ "name": "Month"
},
{
"dataType": "INT",
- "name": "DestWac"
+ "name": "NASDelay"
},
{
"dataType": "STRING",
@@ -230,96 +284,44 @@
"singleValueField": false
},
{
- "dataType": "STRING",
- "name": "TailNum"
- },
- {
- "dataType": "INT",
- "name": "Year"
- },
- {
- "dataType": "STRING",
- "name": "UniqueCarrier"
- },
- {
- "dataType": "STRING",
- "name": "FlightDate"
- },
- {
- "dataType": "INT",
- "name": "DivAirportIDs",
- "singleValueField": false
- },
- {
"dataType": "INT",
- "name": "DivAirportLandings"
- },
- {
- "dataType": "STRING",
- "name": "DepTimeBlk"
- },
- {
- "dataType": "INT",
- "name": "DivAirportSeqIDs",
- "singleValueField": false
- },
- {
- "dataType": "STRING",
- "name": "DivAirports",
- "singleValueField": false
- },
- {
- "dataType": "INT",
- "name": "DivReachedDest"
+ "name": "SecurityDelay"
},
{
"dataType": "STRING",
- "name": "DivTailNums",
- "singleValueField": false
- },
- {
- "dataType": "INT",
- "name": "Diverted"
- },
- {
- "dataType": "INT",
- "name": "FlightNum"
+ "name": "TailNum"
},
{
"dataType": "INT",
- "name": "Month"
+ "name": "TaxiIn"
},
{
"dataType": "INT",
- "name": "DistanceGroup"
+ "name": "TaxiOut"
},
{
"dataType": "INT",
- "name": "AirlineID"
+ "name": "Year"
},
{
"dataType": "INT",
- "name": "DivLongestGTimes",
- "singleValueField": false
+ "name": "WheelsOn"
},
{
"dataType": "INT",
- "name": "DivTotalGTimes",
- "singleValueField": false
+ "name": "WheelsOff"
},
{
"dataType": "INT",
- "name": "DivWheelsOffs",
- "singleValueField": false
+ "name": "WeatherDelay"
},
{
"dataType": "STRING",
- "name": "ArrTimeBlk"
+ "name": "UniqueCarrier"
},
{
"dataType": "INT",
- "name": "DivWheelsOns",
- "singleValueField": false
+ "name": "TotalAddGTime"
}
],
"timeFieldSpec": {
@@ -330,4 +332,4 @@
}
},
"schemaName": "airlineStats"
-}
\ No newline at end of file
+}
diff --git a/pinot-tools/src/main/resources/examples/stream/airlineStats/airlineStats_schema.json b/pinot-tools/src/main/resources/examples/stream/airlineStats/airlineStats_schema.json
index 0c058c5..02143ca 100644
--- a/pinot-tools/src/main/resources/examples/stream/airlineStats/airlineStats_schema.json
+++ b/pinot-tools/src/main/resources/examples/stream/airlineStats/airlineStats_schema.json
@@ -1,14 +1,21 @@
{
- "metricFieldSpecs": [{
- "dataType": "INT",
- "name": "ActualElapsedTime"
- },
+ "metricFieldSpecs": [
+ ],
+ "dimensionFieldSpecs": [
+ {
+ "dataType": "INT",
+ "name": "ActualElapsedTime"
+ },
{
"dataType": "INT",
"name": "AirTime"
},
{
"dataType": "INT",
+ "name": "AirlineID"
+ },
+ {
+ "dataType": "INT",
"name": "ArrDel15"
},
{
@@ -24,6 +31,10 @@
"name": "ArrTime"
},
{
+ "dataType": "STRING",
+ "name": "ArrTimeBlk"
+ },
+ {
"dataType": "INT",
"name": "ArrivalDelayGroups"
},
@@ -40,11 +51,31 @@
"name": "CRSElapsedTime"
},
{
+ "dataType": "STRING",
+ "name": "CancellationCode"
+ },
+ {
+ "dataType": "INT",
+ "name": "Cancelled"
+ },
+ {
+ "dataType": "STRING",
+ "name": "Carrier"
+ },
+ {
"dataType": "INT",
"name": "CarrierDelay"
},
{
"dataType": "INT",
+ "name": "DayOfWeek"
+ },
+ {
+ "dataType": "INT",
+ "name": "DayofMonth"
+ },
+ {
+ "dataType": "INT",
"name": "DepDel15"
},
{
@@ -60,129 +91,152 @@
"name": "DepTime"
},
{
+ "dataType": "STRING",
+ "name": "DepTimeBlk"
+ },
+ {
"dataType": "INT",
"name": "DepartureDelayGroups"
},
{
- "dataType": "INT",
- "name": "Distance"
+ "dataType": "STRING",
+ "name": "Dest"
},
{
"dataType": "INT",
- "name": "DivActualElapsedTime"
+ "name": "DestAirportID"
},
{
"dataType": "INT",
- "name": "DivArrDelay"
+ "name": "DestAirportSeqID"
},
{
"dataType": "INT",
- "name": "DivDistance"
+ "name": "DestCityMarketID"
},
{
- "dataType": "INT",
- "name": "FirstDepTime"
+ "dataType": "STRING",
+ "name": "DestCityName"
+ },
+ {
+ "dataType": "STRING",
+ "name": "DestState"
},
{
"dataType": "INT",
- "name": "Flights"
+ "name": "DestStateFips"
+ },
+ {
+ "dataType": "STRING",
+ "name": "DestStateName"
},
{
"dataType": "INT",
- "name": "LateAircraftDelay"
+ "name": "DestWac"
},
{
"dataType": "INT",
- "name": "LongestAddGTime"
+ "name": "Distance"
},
{
"dataType": "INT",
- "name": "NASDelay"
+ "name": "DistanceGroup"
},
{
"dataType": "INT",
- "name": "SecurityDelay"
+ "name": "DivActualElapsedTime"
},
{
"dataType": "INT",
- "name": "TaxiIn"
+ "name": "DivAirportIDs",
+ "singleValueField": false
},
{
"dataType": "INT",
- "name": "TaxiOut"
+ "name": "DivAirportLandings"
},
{
"dataType": "INT",
- "name": "WheelsOn"
+ "name": "DivAirportSeqIDs",
+ "singleValueField": false
+ },
+ {
+ "dataType": "STRING",
+ "name": "DivAirports",
+ "singleValueField": false
},
{
"dataType": "INT",
- "name": "WheelsOff"
+ "name": "DivArrDelay"
},
{
"dataType": "INT",
- "name": "WeatherDelay"
+ "name": "DivDistance"
},
{
"dataType": "INT",
- "name": "TotalAddGTime"
- }
- ],
- "dimensionFieldSpecs": [{
- "dataType": "STRING",
- "name": "CancellationCode"
- },
+ "name": "DivLongestGTimes",
+ "singleValueField": false
+ },
{
"dataType": "INT",
- "name": "Cancelled"
+ "name": "DivReachedDest"
},
{
"dataType": "STRING",
- "name": "Carrier"
+ "name": "DivTailNums",
+ "singleValueField": false
},
{
"dataType": "INT",
- "name": "DayOfWeek"
+ "name": "DivTotalGTimes",
+ "singleValueField": false
},
{
"dataType": "INT",
- "name": "DayofMonth"
+ "name": "DivWheelsOffs",
+ "singleValueField": false
},
{
- "dataType": "STRING",
- "name": "Dest"
+ "dataType": "INT",
+ "name": "DivWheelsOns",
+ "singleValueField": false
},
{
"dataType": "INT",
- "name": "DestAirportID"
+ "name": "Diverted"
},
{
"dataType": "INT",
- "name": "DestAirportSeqID"
+ "name": "FirstDepTime"
+ },
+ {
+ "dataType": "STRING",
+ "name": "FlightDate"
},
{
"dataType": "INT",
- "name": "DestCityMarketID"
+ "name": "FlightNum"
},
{
- "dataType": "STRING",
- "name": "DestCityName"
+ "dataType": "INT",
+ "name": "Flights"
},
{
- "dataType": "STRING",
- "name": "DestState"
+ "dataType": "INT",
+ "name": "LateAircraftDelay"
},
{
"dataType": "INT",
- "name": "DestStateFips"
+ "name": "LongestAddGTime"
},
{
- "dataType": "STRING",
- "name": "DestStateName"
+ "dataType": "INT",
+ "name": "Month"
},
{
"dataType": "INT",
- "name": "DestWac"
+ "name": "NASDelay"
},
{
"dataType": "STRING",
@@ -230,96 +284,44 @@
"singleValueField": false
},
{
- "dataType": "STRING",
- "name": "TailNum"
- },
- {
- "dataType": "INT",
- "name": "Year"
- },
- {
- "dataType": "STRING",
- "name": "UniqueCarrier"
- },
- {
- "dataType": "STRING",
- "name": "FlightDate"
- },
- {
- "dataType": "INT",
- "name": "DivAirportIDs",
- "singleValueField": false
- },
- {
"dataType": "INT",
- "name": "DivAirportLandings"
- },
- {
- "dataType": "STRING",
- "name": "DepTimeBlk"
- },
- {
- "dataType": "INT",
- "name": "DivAirportSeqIDs",
- "singleValueField": false
- },
- {
- "dataType": "STRING",
- "name": "DivAirports",
- "singleValueField": false
- },
- {
- "dataType": "INT",
- "name": "DivReachedDest"
+ "name": "SecurityDelay"
},
{
"dataType": "STRING",
- "name": "DivTailNums",
- "singleValueField": false
- },
- {
- "dataType": "INT",
- "name": "Diverted"
- },
- {
- "dataType": "INT",
- "name": "FlightNum"
+ "name": "TailNum"
},
{
"dataType": "INT",
- "name": "Month"
+ "name": "TaxiIn"
},
{
"dataType": "INT",
- "name": "DistanceGroup"
+ "name": "TaxiOut"
},
{
"dataType": "INT",
- "name": "AirlineID"
+ "name": "Year"
},
{
"dataType": "INT",
- "name": "DivLongestGTimes",
- "singleValueField": false
+ "name": "WheelsOn"
},
{
"dataType": "INT",
- "name": "DivTotalGTimes",
- "singleValueField": false
+ "name": "WheelsOff"
},
{
"dataType": "INT",
- "name": "DivWheelsOffs",
- "singleValueField": false
+ "name": "WeatherDelay"
},
{
"dataType": "STRING",
- "name": "ArrTimeBlk"
+ "name": "UniqueCarrier"
},
{
"dataType": "INT",
- "name": "DivWheelsOns",
- "singleValueField": false
+ "name": "TotalAddGTime"
}
],
"timeFieldSpec": {
@@ -330,4 +332,4 @@
}
},
"schemaName": "airlineStats"
-}
\ No newline at end of file
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org