You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by ap...@apache.org on 2020/06/04 18:16:25 UTC
[incubator-pinot] branch revert-5491-pinot-mock-data-deterministic
created (now 8907ac3)
This is an automated email from the ASF dual-hosted git repository.
apucher pushed a change to branch revert-5491-pinot-mock-data-deterministic
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git.
at 8907ac3 Revert "Enhance GenerateData to produce deterministic time-series (#5491)"
This branch includes the following new commits:
new 8907ac3 Revert "Enhance GenerateData to produce deterministic time-series (#5491)"
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org
[incubator-pinot] 01/01: Revert "Enhance GenerateData to produce
deterministic time-series (#5491)"
Posted by ap...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
apucher pushed a commit to branch revert-5491-pinot-mock-data-deterministic
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
commit 8907ac3ba1a16e54f5814145daa41a2ebef263f3
Author: Alexander Pucher <ap...@apache.org>
AuthorDate: Thu Jun 4 11:16:17 2020 -0700
Revert "Enhance GenerateData to produce deterministic time-series (#5491)"
This reverts commit 5ebcacf6834daf7eb4a3f5b7ca72b6c655aee383.
---
docker/images/pinot/bin/generator.sh | 43 +++-----
.../data/generator/PatternSeasonalGenerator.java | 6 +-
.../data/generator/PatternSpikeGenerator.java | 42 ++------
.../generator/complexWebsite_generator.json | 108 ++++++++++-----------
.../src/main/resources/generator/generator.sh | 76 +++++++++++++++
5 files changed, 154 insertions(+), 121 deletions(-)
diff --git a/docker/images/pinot/bin/generator.sh b/docker/images/pinot/bin/generator.sh
index a79afa1..dc84967 100755
--- a/docker/images/pinot/bin/generator.sh
+++ b/docker/images/pinot/bin/generator.sh
@@ -20,42 +20,35 @@
JAR_PATH="$(find /opt/pinot/lib/pinot-all-*-jar-with-dependencies.jar)"
ADMIN_PATH="/opt/pinot/bin/pinot-admin.sh"
-TEMP_DIR=$(mktemp -d -t pinotGenerator-XXXXXXXX)
-TEMPLATE_BASEDIR="$TEMP_DIR/generator"
+TEMPLATE_BASEDIR="/tmp/pinotGenerator/generator"
+TEMP_DIR="/tmp/pinotGenerator"
-TEMPLATE_NAME="$1"
-if [ -z "$TEMPLATE_NAME" ]; then
+if [ -z "$1" ]; then
echo "No template name specified. Aborting."
exit 1
fi
-TABLE_NAME="$2"
-if [ -z "$TABLE_NAME" ]; then
- echo "No table name specified. Defaulting to '$TEMPLATE_NAME'"
- TABLE_NAME=$TEMPLATE_NAME
-fi
-
+TEMPLATE_NAME="$1"
DATA_DIR="${TEMP_DIR:?}/${TEMPLATE_NAME}"
SEGMENT_DIR="${TEMP_DIR:?}/${TEMPLATE_NAME}Segment"
-echo "Extracting template files to '${TEMP_DIR}'"
-/bin/sh -c "cd \"${TEMP_DIR}\" && jar -xf \"${JAR_PATH}\" \"generator/${TEMPLATE_NAME}_schema.json\" \"generator/${TEMPLATE_NAME}_config.json\" \"generator/${TEMPLATE_NAME}_generator.json\""
+echo "Preparing temp directory for ${TEMPLATE_NAME}"
+rm -rf "${DATA_DIR}"
+rm -rf "${SEGMENT_DIR}"
+mkdir -p "${TEMP_DIR}"
-echo "Setting table name and schema name to $TABLE_NAME"
-sed -i -e "s/\"tableName\": \"$TEMPLATE_NAME\"/\"tableName\": \"$TABLE_NAME\"/g" "${TEMPLATE_BASEDIR}/${TEMPLATE_NAME}_config.json"
-sed -i -e "s/\"schemaName\": \"$TEMPLATE_NAME\"/\"schemaName\": \"$TABLE_NAME\"/g" "${TEMPLATE_BASEDIR}/${TEMPLATE_NAME}_config.json"
-sed -i -e "s/\"schemaName\": \"$TEMPLATE_NAME\"/\"schemaName\": \"$TABLE_NAME\"/g" "${TEMPLATE_BASEDIR}/${TEMPLATE_NAME}_schema.json"
+echo "Extracting template files"
+/bin/sh -c "cd \"${TEMP_DIR}\" && jar -xf \"${JAR_PATH}\" \"generator/${TEMPLATE_NAME}_schema.json\" \"generator/${TEMPLATE_NAME}_config.json\" \"generator/${TEMPLATE_NAME}_generator.json\""
echo "Generating data for ${TEMPLATE_NAME} in ${DATA_DIR}"
${ADMIN_PATH} GenerateData \
--numFiles 1 -numRecords 631152 -format csv \
+-numFiles 1 -numRecords 354780 -format csv \
-schemaFile "${TEMPLATE_BASEDIR}/${TEMPLATE_NAME}_schema.json" \
-schemaAnnotationFile "${TEMPLATE_BASEDIR}/${TEMPLATE_NAME}_generator.json" \
-outDir "$DATA_DIR"
if [ ! -d "${DATA_DIR}" ]; then
echo "Data generation failed. Aborting."
- rm -rf "$TEMP_DIR"
exit 1
fi
@@ -65,25 +58,21 @@ ${ADMIN_PATH} CreateSegment \
-tableConfigFile "${TEMPLATE_BASEDIR}/${TEMPLATE_NAME}_config.json" \
-schemaFile "${TEMPLATE_BASEDIR}/${TEMPLATE_NAME}_schema.json" \
-dataDir "${DATA_DIR}" \
--outDir "${SEGMENT_DIR}"
+-outDir "${SEGMENT_DIR}" || exit 1
if [ ! -d "${SEGMENT_DIR}" ]; then
echo "Data generation failed. Aborting."
- rm -rf "$TEMP_DIR"
exit 1
fi
-echo "Adding table ${TABLE_NAME} from template ${TEMPLATE_NAME}"
+echo "Adding table ${TEMPLATE_NAME}"
${ADMIN_PATH} AddTable -exec \
-tableConfigFile "${TEMPLATE_BASEDIR}/${TEMPLATE_NAME}_config.json" \
-schemaFile "${TEMPLATE_BASEDIR}/${TEMPLATE_NAME}_schema.json" || exit 1
echo "Uploading segment for ${TEMPLATE_NAME}"
${ADMIN_PATH} UploadSegment \
--tableName "${TABLE_NAME}" \
--segmentDir "${SEGMENT_DIR}"
-
-echo "Deleting temp directory"
-rm -rf "$TEMP_DIR"
+-tableName "${TEMPLATE_NAME}" \
+-segmentDir "${SEGMENT_DIR}" || exit 1
-echo "Succesfully created table ${TABLE_NAME} from template ${TEMPLATE_NAME}"
+echo "Succesfully applied template ${TEMPLATE_NAME}"
diff --git a/pinot-tools/src/main/java/org/apache/pinot/tools/data/generator/PatternSeasonalGenerator.java b/pinot-tools/src/main/java/org/apache/pinot/tools/data/generator/PatternSeasonalGenerator.java
index 38dcd81..36aa600 100644
--- a/pinot-tools/src/main/java/org/apache/pinot/tools/data/generator/PatternSeasonalGenerator.java
+++ b/pinot-tools/src/main/java/org/apache/pinot/tools/data/generator/PatternSeasonalGenerator.java
@@ -20,7 +20,6 @@ package org.apache.pinot.tools.data.generator;
import org.apache.commons.configuration.PropertyConverter;
import org.apache.commons.math3.distribution.NormalDistribution;
-import org.apache.commons.math3.random.Well19937c;
import java.util.Arrays;
import java.util.List;
@@ -65,19 +64,18 @@ public class PatternSeasonalGenerator implements Generator {
PropertyConverter.toDouble(templateConfig.getOrDefault("wavelength", 0)),
PropertyConverter. toDouble(templateConfig.getOrDefault("amplitude", 0)),
PropertyConverter. toDouble(templateConfig.getOrDefault("offset", 0)),
- PropertyConverter. toInteger(templateConfig.getOrDefault("seed", 0)),
toDoubleArray(templateConfig.get("scalingFactors"), 1));
}
public PatternSeasonalGenerator(double mean, double sigma, double trend, double wavelength, double amplitude,
- double offset, int seed, double[] scalingFactors) {
+ double offset, double[] scalingFactors) {
this.trend = trend;
this.wavelength = wavelength;
this.amplitude = amplitude;
this.offset = offset;
this.scalingFactors = scalingFactors;
- this.generator = new NormalDistribution(new Well19937c(seed), mean, sigma, 1.0E-9D);
+ this.generator = new NormalDistribution(mean, sigma);
}
@Override
diff --git a/pinot-tools/src/main/java/org/apache/pinot/tools/data/generator/PatternSpikeGenerator.java b/pinot-tools/src/main/java/org/apache/pinot/tools/data/generator/PatternSpikeGenerator.java
index 5b23e09..570551a 100644
--- a/pinot-tools/src/main/java/org/apache/pinot/tools/data/generator/PatternSpikeGenerator.java
+++ b/pinot-tools/src/main/java/org/apache/pinot/tools/data/generator/PatternSpikeGenerator.java
@@ -19,11 +19,7 @@
package org.apache.pinot.tools.data.generator;
import org.apache.commons.configuration.PropertyConverter;
-import org.apache.commons.math3.distribution.AbstractRealDistribution;
-import org.apache.commons.math3.distribution.ExponentialDistribution;
import org.apache.commons.math3.distribution.LogNormalDistribution;
-import org.apache.commons.math3.distribution.UniformRealDistribution;
-import org.apache.commons.math3.random.Well19937c;
import java.util.Map;
@@ -50,60 +46,34 @@ public class PatternSpikeGenerator implements Generator {
private final double baseline;
private final double smoothing;
- private final AbstractRealDistribution arrivalGenerator;
- private final AbstractRealDistribution magnitudeGenerator;
+ private final LogNormalDistribution arrivalGenerator;
+ private final LogNormalDistribution magnitudeGenerator;
private long step = -1;
private long nextArrival;
private double lastValue;
- enum DistributionType {
- LOGNORMAL,
- EXPONENTIAL,
- UNIFORM,
- FIXED
- }
-
public PatternSpikeGenerator(Map<String, Object> templateConfig) {
this(PropertyConverter.toDouble(templateConfig.getOrDefault("baseline", 0)),
- DistributionType.valueOf(templateConfig.getOrDefault("arrivalType", "lognormal").toString().toUpperCase()),
PropertyConverter.toDouble(templateConfig.getOrDefault("arrivalMean", 2)),
PropertyConverter.toDouble(templateConfig.getOrDefault("arrivalSigma", 1)),
- DistributionType.valueOf(templateConfig.getOrDefault("magnitudeType", "lognormal").toString().toUpperCase()),
PropertyConverter.toDouble(templateConfig.getOrDefault("magnitudeMean", 2)),
PropertyConverter.toDouble(templateConfig.getOrDefault("magnitudeSigma", 1)),
- PropertyConverter.toDouble(templateConfig.getOrDefault("smoothing", 0)),
- PropertyConverter.toInteger(templateConfig.getOrDefault("seed", 0)));
+ PropertyConverter.toDouble(templateConfig.getOrDefault("smoothing", 0)));
}
- public PatternSpikeGenerator(double baseline, DistributionType arrivalType, double arrivalMean, double arrivalSigma,
- DistributionType magnitudeType, double magnitudeMean, double magnitudeSigma, double smoothing, int seed) {
+ public PatternSpikeGenerator(double baseline, double arrivalMean, double arrivalSigma, double magnitudeMean, double magnitudeSigma, double smoothing) {
this.baseline = baseline;
this.smoothing = smoothing;
- this.arrivalGenerator = makeDist(arrivalType, arrivalMean, arrivalSigma, seed);
- this.magnitudeGenerator = makeDist(magnitudeType, magnitudeMean, magnitudeSigma, seed);
+ this.arrivalGenerator = new LogNormalDistribution(arrivalMean, arrivalSigma);
+ this.magnitudeGenerator = new LogNormalDistribution(magnitudeMean, magnitudeSigma);
this.nextArrival = (long) arrivalGenerator.sample();
this.lastValue = baseline;
}
- private static AbstractRealDistribution makeDist(DistributionType type, double mean, double sigma, int seed) {
- switch (type) {
- case LOGNORMAL:
- return new LogNormalDistribution(new Well19937c(seed), mean, sigma, 1.0E-9D);
- case EXPONENTIAL:
- return new ExponentialDistribution(new Well19937c(seed), mean, 1.0E-9D);
- case UNIFORM:
- return new UniformRealDistribution(new Well19937c(seed), mean - sigma, mean + sigma);
- case FIXED:
- return new UniformRealDistribution(new Well19937c(seed), mean, mean + 1.0E-9D);
- default:
- throw new IllegalArgumentException(String.format("Unsupported distribution type '%s", type));
- }
- }
-
@Override
public void init() {
// left blank
diff --git a/pinot-tools/src/main/resources/generator/complexWebsite_generator.json b/pinot-tools/src/main/resources/generator/complexWebsite_generator.json
index 7777d33..41f08c9 100644
--- a/pinot-tools/src/main/resources/generator/complexWebsite_generator.json
+++ b/pinot-tools/src/main/resources/generator/complexWebsite_generator.json
@@ -31,24 +31,24 @@
"type": "SEASONAL", "wavelength": 24, "scalingFactors": [ 0.4, 0.9, 1.0, 1.0, 1.0, 0.8, 0.4 ]
},
"generatorBins": [
- [ { "mean": 50, "sigma": 4.0, "amplitude": 40, "offset": 0.0, "seed": 0 } ],
- [ { "mean": 25, "sigma": 2.5, "amplitude": 15, "offset": 0.4, "seed": 1 } ],
- [ { "mean": 10, "sigma": 1.0, "amplitude": 5, "offset": 0.6, "seed": 2 } ],
- [ { "mean": 90, "sigma": 6.0, "amplitude": 80, "offset": 0.0, "seed": 3 }, { "type": "SPIKE", "arrivalType": "EXPONENTIAL", "arrivalMean": 120, "arrivalSigma": 1, "magnitudeMean": 5, "magnitudeSigma": 1, "smoothing": 0.7, "seed": 3 } ],
- [ { "mean": 45, "sigma": 3.0, "amplitude": 35, "offset": 0.4, "seed": 4 } ],
- [ { "mean": 20, "sigma": 1.5, "amplitude": 15, "offset": 0.6, "seed": 5 } ],
- [ { "mean": 10, "sigma": 0.5, "amplitude": 8, "offset": 0.0, "seed": 6 } ],
- [ { "mean": 25, "sigma": 2.5, "amplitude": 15, "offset": 0.4, "seed": 7 } ],
- [ { "mean": 10, "sigma": 1.0, "amplitude": 5, "offset": 0.6, "seed": 8 } ],
- [ { "mean": 20, "sigma": 1.0, "amplitude": 18, "offset": 0.0, "seed": 9 }, { "type": "SPIKE", "arrivalType": "EXPONENTIAL", "arrivalMean": 120, "arrivalSigma": 1, "magnitudeMean": 0.5, "magnitudeSigma": 0.5, "smoothing": 0.5, "seed": 3 } ],
- [ { "mean": 45, "sigma": 3.0, "amplitude": 35, "offset": 0.4, "seed": 10 } ],
- [ { "mean": 20, "sigma": 1.5, "amplitude": 15, "offset": 0.6, "seed": 11 } ],
- [ { "mean": 50, "sigma": 4.0, "amplitude": 40, "offset": 0.0, "seed": 12 } ],
- [ { "mean": 10, "sigma": 1.0, "amplitude": 8, "offset": 0.4, "seed": 13 } ],
- [ { "mean": 10, "sigma": 1.0, "amplitude": 5, "offset": 0.6, "seed": 14 } ],
- [ { "mean": 90, "sigma": 6.0, "amplitude": 80, "offset": 0.0, "seed": 15 }, { "type": "SPIKE", "arrivalType": "EXPONENTIAL", "arrivalMean": 120, "arrivalSigma": 1, "magnitudeMean": 4, "magnitudeSigma": 1, "smoothing": 0.5, "seed": 3 } ],
- [ { "mean": 45, "sigma": 3.0, "amplitude": 35, "offset": 0.4, "seed": 16 } ],
- [ { "mean": 10, "sigma": 0.5, "amplitude": 9, "offset": 0.6, "seed": 17 } ]
+ [ { "mean": 50, "sigma": 4.0, "amplitude": 40, "offset": 0.0 } ],
+ [ { "mean": 25, "sigma": 2.5, "amplitude": 15, "offset": 0.4 } ],
+ [ { "mean": 10, "sigma": 1.0, "amplitude": 5, "offset": 0.6 } ],
+ [ { "mean": 90, "sigma": 6.0, "amplitude": 80, "offset": 0.0 }, { "type": "SPIKE", "arrivalMean": 5, "arrivalSigma": 1, "magnitudeMean": 5, "magnitudeSigma": 1, "smoothing": 0.7 } ],
+ [ { "mean": 45, "sigma": 3.0, "amplitude": 35, "offset": 0.4 } ],
+ [ { "mean": 20, "sigma": 1.5, "amplitude": 15, "offset": 0.6 } ],
+ [ { "mean": 10, "sigma": 0.5, "amplitude": 8, "offset": 0.0 } ],
+ [ { "mean": 25, "sigma": 2.5, "amplitude": 15, "offset": 0.4 } ],
+ [ { "mean": 10, "sigma": 1.0, "amplitude": 5, "offset": 0.6 } ],
+ [ { "mean": 20, "sigma": 1.0, "amplitude": 18, "offset": 0.0 }, { "type": "SPIKE", "arrivalMean": 5, "arrivalSigma": 1, "magnitudeMean": 0.5, "magnitudeSigma": 0.5, "smoothing": 0.5 } ],
+ [ { "mean": 45, "sigma": 3.0, "amplitude": 35, "offset": 0.4 } ],
+ [ { "mean": 20, "sigma": 1.5, "amplitude": 15, "offset": 0.6 } ],
+ [ { "mean": 50, "sigma": 4.0, "amplitude": 40, "offset": 0.0 } ],
+ [ { "mean": 10, "sigma": 1.0, "amplitude": 8, "offset": 0.4 } ],
+ [ { "mean": 10, "sigma": 1.0, "amplitude": 5, "offset": 0.6 } ],
+ [ { "mean": 90, "sigma": 6.0, "amplitude": 80, "offset": 0.0 }, { "type": "SPIKE", "arrivalMean": 5, "arrivalSigma": 1, "magnitudeMean": 4, "magnitudeSigma": 1, "smoothing": 0.5 } ],
+ [ { "mean": 45, "sigma": 3.0, "amplitude": 35, "offset": 0.4 } ],
+ [ { "mean": 10, "sigma": 0.5, "amplitude": 9, "offset": 0.6 } ]
]
}
},
@@ -60,24 +60,24 @@
"type": "SEASONAL", "wavelength": 24, "scalingFactors": [ 0.5, 0.9, 1.0, 1.0, 1.0, 0.9, 0.5 ]
},
"generatorBins": [
- [ { "mean": 12, "sigma": 1.0, "amplitude": 10, "offset": 0.0, "seed": 0 } ],
- [ { "mean": 6, "sigma": 0.5, "amplitude": 5, "offset": 0.4, "seed": 1 } ],
- [ { "mean": 2, "sigma": 0.3, "amplitude": 2, "offset": 0.6, "seed": 2 } ],
- [ { "mean": 20, "sigma": 2.0, "amplitude": 18, "offset": 0.0, "seed": 3 }, { "type": "SPIKE", "arrivalType": "EXPONENTIAL", "arrivalMean": 120, "arrivalSigma": 1, "magnitudeMean": 2, "magnitudeSigma": 1, "smoothing": 0.5, "seed": 3 } ],
- [ { "mean": 9, "sigma": 1.0, "amplitude": 7, "offset": 0.4, "seed": 4 } ],
- [ { "mean": 4, "sigma": 0.5, "amplitude": 3, "offset": 0.6, "seed": 5 } ],
- [ { "mean": 3, "sigma": 0.5, "amplitude": 2, "offset": 0.0, "seed": 6 } ],
- [ { "mean": 6, "sigma": 0.5, "amplitude": 5, "offset": 0.4, "seed": 7 } ],
- [ { "mean": 2, "sigma": 0.3, "amplitude": 2, "offset": 0.6, "seed": 8 } ],
- [ { "mean": 5, "sigma": 0.5, "amplitude": 4, "offset": 0.0, "seed": 9 }, { "type": "SPIKE", "arrivalType": "EXPONENTIAL", "arrivalMean": 120, "arrivalSigma": 1, "magnitudeMean": 0.5, "magnitudeSigma": 0.5, "smoothing": 0.3, "seed": 3 } ],
- [ { "mean": 9, "sigma": 1.0, "amplitude": 7, "offset": 0.4, "seed": 10 } ],
- [ { "mean": 4, "sigma": 0.5, "amplitude": 3, "offset": 0.6, "seed": 11 } ],
- [ { "mean": 12, "sigma": 1.0, "amplitude": 10, "offset": 0.0, "seed": 12 } ],
- [ { "mean": 6, "sigma": 0.5, "amplitude": 5, "offset": 0.4, "seed": 13 } ],
- [ { "mean": 2, "sigma": 0.3, "amplitude": 2, "offset": 0.6, "seed": 14 } ],
- [ { "mean": 20, "sigma": 2.0, "amplitude": 18, "offset": 0.0, "seed": 15 }, { "type": "SPIKE", "arrivalType": "EXPONENTIAL", "arrivalMean": 120, "arrivalSigma": 1, "magnitudeMean": 3, "magnitudeSigma": 1, "smoothing": 0.3, "seed": 3 } ],
- [ { "mean": 9, "sigma": 1.0, "amplitude": 7, "offset": 0.4, "seed": 16 } ],
- [ { "mean": 4, "sigma": 0.5, "amplitude": 3, "offset": 0.6, "seed": 17 } ]
+ [ { "mean": 12, "sigma": 1.0, "amplitude": 10, "offset": 0.0 } ],
+ [ { "mean": 6, "sigma": 0.5, "amplitude": 5, "offset": 0.4 } ],
+ [ { "mean": 2, "sigma": 0.3, "amplitude": 2, "offset": 0.6 } ],
+ [ { "mean": 20, "sigma": 2.0, "amplitude": 18, "offset": 0.0 }, { "type": "SPIKE", "arrivalMean": 4, "arrivalSigma": 1, "magnitudeMean": 2, "magnitudeSigma": 1, "smoothing": 0.5 } ],
+ [ { "mean": 9, "sigma": 1.0, "amplitude": 7, "offset": 0.4 } ],
+ [ { "mean": 4, "sigma": 0.5, "amplitude": 3, "offset": 0.6 } ],
+ [ { "mean": 3, "sigma": 0.5, "amplitude": 2, "offset": 0.0 } ],
+ [ { "mean": 6, "sigma": 0.5, "amplitude": 5, "offset": 0.4 } ],
+ [ { "mean": 2, "sigma": 0.3, "amplitude": 2, "offset": 0.6 } ],
+ [ { "mean": 5, "sigma": 0.5, "amplitude": 4, "offset": 0.0 }, { "type": "SPIKE", "arrivalMean": 5, "arrivalSigma": 1, "magnitudeMean": 0.5, "magnitudeSigma": 0.5, "smoothing": 0.3 } ],
+ [ { "mean": 9, "sigma": 1.0, "amplitude": 7, "offset": 0.4 } ],
+ [ { "mean": 4, "sigma": 0.5, "amplitude": 3, "offset": 0.6 } ],
+ [ { "mean": 12, "sigma": 1.0, "amplitude": 10, "offset": 0.0 } ],
+ [ { "mean": 6, "sigma": 0.5, "amplitude": 5, "offset": 0.4 } ],
+ [ { "mean": 2, "sigma": 0.3, "amplitude": 2, "offset": 0.6 } ],
+ [ { "mean": 20, "sigma": 2.0, "amplitude": 18, "offset": 0.0 }, { "type": "SPIKE", "arrivalMean": 5, "arrivalSigma": 1, "magnitudeMean": 3, "magnitudeSigma": 1, "smoothing": 0.3 } ],
+ [ { "mean": 9, "sigma": 1.0, "amplitude": 7, "offset": 0.4 } ],
+ [ { "mean": 4, "sigma": 0.5, "amplitude": 3, "offset": 0.6 } ]
]
}
},
@@ -89,24 +89,24 @@
"type": "SPIKE", "arrivalMean": 4, "arrivalSigma": 1, "magnitudeMean": 2, "magnitudeSigma": 1, "smoothing": 0.3
},
"generatorBins": [
- [ { "arrivalMean": 4, "magnitudeMean": 2.0, "seed": 0 } ],
- [ { "arrivalMean": 3, "magnitudeMean": 1.0, "seed": 1 } ],
- [ { "arrivalMean": 5, "magnitudeMean": 0.2, "seed": 2 } ],
- [ { "arrivalMean": 4, "magnitudeMean": 2.5, "seed": 3 } ],
- [ { "arrivalMean": 3, "magnitudeMean": 0.8, "seed": 4 } ],
- [ { "arrivalMean": 5, "magnitudeMean": 0.1, "seed": 5 } ],
- [ { "arrivalMean": 1, "magnitudeMean": 0.5, "seed": 6 } ],
- [ { "arrivalMean": 3, "magnitudeMean": 1.0, "seed": 7 } ],
- [ { "arrivalMean": 5, "magnitudeMean": 0.2, "seed": 8 } ],
- [ { "arrivalMean": 1, "magnitudeMean": 0.5, "seed": 9 } ],
- [ { "arrivalMean": 3, "magnitudeMean": 0.8, "seed": 10 } ],
- [ { "arrivalMean": 5, "magnitudeMean": 0.1, "seed": 11 } ],
- [ { "arrivalMean": 4, "magnitudeMean": 2.0, "seed": 12 } ],
- [ { "arrivalMean": 3, "magnitudeMean": 1.0, "seed": 13 } ],
- [ { "arrivalMean": 5, "magnitudeMean": 0.2, "seed": 14 } ],
- [ { "arrivalMean": 4, "magnitudeMean": 2.5, "seed": 15 } ],
- [ { "arrivalMean": 3, "magnitudeMean": 0.8, "seed": 16 } ],
- [ { "arrivalMean": 5, "magnitudeMean": 0.1, "seed": 17 } ]
+ [ { "arrivalMean": 4, "magnitudeMean": 2.0 } ],
+ [ { "arrivalMean": 3, "magnitudeMean": 1.0 } ],
+ [ { "arrivalMean": 5, "magnitudeMean": 0.2 } ],
+ [ { "arrivalMean": 4, "magnitudeMean": 2.5 } ],
+ [ { "arrivalMean": 3, "magnitudeMean": 0.8 } ],
+ [ { "arrivalMean": 5, "magnitudeMean": 0.1 } ],
+ [ { "arrivalMean": 1, "magnitudeMean": 0.5 } ],
+ [ { "arrivalMean": 3, "magnitudeMean": 1.0 } ],
+ [ { "arrivalMean": 5, "magnitudeMean": 0.2 } ],
+ [ { "arrivalMean": 1, "magnitudeMean": 0.5 } ],
+ [ { "arrivalMean": 3, "magnitudeMean": 0.8 } ],
+ [ { "arrivalMean": 5, "magnitudeMean": 0.1 } ],
+ [ { "arrivalMean": 4, "magnitudeMean": 2.0 } ],
+ [ { "arrivalMean": 3, "magnitudeMean": 1.0 } ],
+ [ { "arrivalMean": 5, "magnitudeMean": 0.2 } ],
+ [ { "arrivalMean": 4, "magnitudeMean": 2.5 } ],
+ [ { "arrivalMean": 3, "magnitudeMean": 0.8 } ],
+ [ { "arrivalMean": 5, "magnitudeMean": 0.1 } ]
]
}
}
diff --git a/pinot-tools/src/main/resources/generator/generator.sh b/pinot-tools/src/main/resources/generator/generator.sh
new file mode 100755
index 0000000..ac1e746
--- /dev/null
+++ b/pinot-tools/src/main/resources/generator/generator.sh
@@ -0,0 +1,76 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+
+#ADMIN_PATH="/opt/pinot/bin/pinot-admin.sh"
+ADMIN_PATH="./pinot-tools/target/pinot-tools-pkg/bin/pinot-admin.sh"
+#PATTERN_BASEDIR="/opt/pinot/examples/docker/generators"
+PATTERN_BASEDIR="./pinot-tools/src/main/resources/generator"
+TEMP_DIR="/tmp/pinotGenerator"
+
+if [ -z "$1" ]; then
+ echo "No PATTERN name specified. Aborting."
+ exit 1
+fi
+
+PATTERN_NAME="$1"
+DATA_DIR="${TEMP_DIR:?}/${PATTERN_NAME}"
+SEGMENT_DIR="${TEMP_DIR:?}/${PATTERN_NAME}Segment"
+
+echo "Preparing temp directory for ${PATTERN_NAME}"
+rm -rf "${DATA_DIR}"
+rm -rf "${SEGMENT_DIR}"
+mkdir -p "${TEMP_DIR}"
+
+echo "Generating data for ${PATTERN_NAME} in ${DATA_DIR}"
+${ADMIN_PATH} GenerateData \
+-numFiles 1 -numRecords 354780 -format csv \
+-schemaFile "${PATTERN_BASEDIR}/${PATTERN_NAME}_schema.json" \
+-schemaAnnotationFile "${PATTERN_BASEDIR}/${PATTERN_NAME}_generator.json" \
+-outDir "$DATA_DIR"
+
+if [ ! -d "${DATA_DIR}" ]; then
+ echo "Data generation failed. Aborting."
+ exit 1
+fi
+
+echo "Creating segment for ${PATTERN_NAME} in ${SEGMENT_DIR}"
+${ADMIN_PATH} CreateSegment \
+-tableName "${PATTERN_NAME}" -segmentName "${PATTERN_NAME}" -format CSV -overwrite \
+-schemaFile "${PATTERN_BASEDIR}/${PATTERN_NAME}_schema.json" \
+-dataDir "${DATA_DIR}" \
+-outDir "${SEGMENT_DIR}" || exit 1
+
+if [ ! -d "${SEGMENT_DIR}" ]; then
+ echo "Data generation failed. Aborting."
+ exit 1
+fi
+
+echo "Adding table ${PATTERN_NAME}"
+${ADMIN_PATH} AddTable -exec \
+-tableConfigFile "${PATTERN_BASEDIR}/${PATTERN_NAME}_config.json" \
+-schemaFile "${PATTERN_BASEDIR}/${PATTERN_NAME}_schema.json" || exit 1
+
+echo "Uploading segment for ${PATTERN_NAME}"
+${ADMIN_PATH} UploadSegment \
+-tableName "${PATTERN_NAME}" \
+-segmentDir "${SEGMENT_DIR}" || exit 1
+
+echo "Succesfully applied PATTERN ${PATTERN_NAME}"
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org