You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by ap...@apache.org on 2020/06/03 23:09:10 UTC
[incubator-pinot] 02/02: table name support in generator.sh
This is an automated email from the ASF dual-hosted git repository.
apucher pushed a commit to branch pinot-mock-data-deterministic
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
commit cbab97f6a975ad77a2b8e9b6b4fd191f0272d8a8
Author: Alexander Pucher <al...@alexpucher.com>
AuthorDate: Wed Jun 3 16:07:28 2020 -0700
table name support in generator.sh
---
docker/images/pinot/bin/generator.sh | 43 +++++++-----
.../src/main/resources/generator/generator.sh | 76 ----------------------
2 files changed, 27 insertions(+), 92 deletions(-)
diff --git a/docker/images/pinot/bin/generator.sh b/docker/images/pinot/bin/generator.sh
index dc84967..a79afa1 100755
--- a/docker/images/pinot/bin/generator.sh
+++ b/docker/images/pinot/bin/generator.sh
@@ -20,35 +20,42 @@
JAR_PATH="$(find /opt/pinot/lib/pinot-all-*-jar-with-dependencies.jar)"
ADMIN_PATH="/opt/pinot/bin/pinot-admin.sh"
-TEMPLATE_BASEDIR="/tmp/pinotGenerator/generator"
-TEMP_DIR="/tmp/pinotGenerator"
+TEMP_DIR=$(mktemp -d -t pinotGenerator-XXXXXXXX)
+TEMPLATE_BASEDIR="$TEMP_DIR/generator"
-if [ -z "$1" ]; then
+TEMPLATE_NAME="$1"
+if [ -z "$TEMPLATE_NAME" ]; then
echo "No template name specified. Aborting."
exit 1
fi
-TEMPLATE_NAME="$1"
+TABLE_NAME="$2"
+if [ -z "$TABLE_NAME" ]; then
+ echo "No table name specified. Defaulting to '$TEMPLATE_NAME'"
+ TABLE_NAME=$TEMPLATE_NAME
+fi
+
DATA_DIR="${TEMP_DIR:?}/${TEMPLATE_NAME}"
SEGMENT_DIR="${TEMP_DIR:?}/${TEMPLATE_NAME}Segment"
-echo "Preparing temp directory for ${TEMPLATE_NAME}"
-rm -rf "${DATA_DIR}"
-rm -rf "${SEGMENT_DIR}"
-mkdir -p "${TEMP_DIR}"
-
-echo "Extracting template files"
+echo "Extracting template files to '${TEMP_DIR}'"
/bin/sh -c "cd \"${TEMP_DIR}\" && jar -xf \"${JAR_PATH}\" \"generator/${TEMPLATE_NAME}_schema.json\" \"generator/${TEMPLATE_NAME}_config.json\" \"generator/${TEMPLATE_NAME}_generator.json\""
+echo "Setting table name and schema name to $TABLE_NAME"
+sed -i -e "s/\"tableName\": \"$TEMPLATE_NAME\"/\"tableName\": \"$TABLE_NAME\"/g" "${TEMPLATE_BASEDIR}/${TEMPLATE_NAME}_config.json"
+sed -i -e "s/\"schemaName\": \"$TEMPLATE_NAME\"/\"schemaName\": \"$TABLE_NAME\"/g" "${TEMPLATE_BASEDIR}/${TEMPLATE_NAME}_config.json"
+sed -i -e "s/\"schemaName\": \"$TEMPLATE_NAME\"/\"schemaName\": \"$TABLE_NAME\"/g" "${TEMPLATE_BASEDIR}/${TEMPLATE_NAME}_schema.json"
+
echo "Generating data for ${TEMPLATE_NAME} in ${DATA_DIR}"
${ADMIN_PATH} GenerateData \
--numFiles 1 -numRecords 354780 -format csv \
+-numFiles 1 -numRecords 631152 -format csv \
-schemaFile "${TEMPLATE_BASEDIR}/${TEMPLATE_NAME}_schema.json" \
-schemaAnnotationFile "${TEMPLATE_BASEDIR}/${TEMPLATE_NAME}_generator.json" \
-outDir "$DATA_DIR"
if [ ! -d "${DATA_DIR}" ]; then
echo "Data generation failed. Aborting."
+ rm -rf "$TEMP_DIR"
exit 1
fi
@@ -58,21 +65,25 @@ ${ADMIN_PATH} CreateSegment \
-tableConfigFile "${TEMPLATE_BASEDIR}/${TEMPLATE_NAME}_config.json" \
-schemaFile "${TEMPLATE_BASEDIR}/${TEMPLATE_NAME}_schema.json" \
-dataDir "${DATA_DIR}" \
--outDir "${SEGMENT_DIR}" || exit 1
+-outDir "${SEGMENT_DIR}"
if [ ! -d "${SEGMENT_DIR}" ]; then
echo "Data generation failed. Aborting."
+ rm -rf "$TEMP_DIR"
exit 1
fi
-echo "Adding table ${TEMPLATE_NAME}"
+echo "Adding table ${TABLE_NAME} from template ${TEMPLATE_NAME}"
${ADMIN_PATH} AddTable -exec \
-tableConfigFile "${TEMPLATE_BASEDIR}/${TEMPLATE_NAME}_config.json" \
-schemaFile "${TEMPLATE_BASEDIR}/${TEMPLATE_NAME}_schema.json" || exit 1
echo "Uploading segment for ${TEMPLATE_NAME}"
${ADMIN_PATH} UploadSegment \
--tableName "${TEMPLATE_NAME}" \
--segmentDir "${SEGMENT_DIR}" || exit 1
+-tableName "${TABLE_NAME}" \
+-segmentDir "${SEGMENT_DIR}"
+
+echo "Deleting temp directory"
+rm -rf "$TEMP_DIR"
-echo "Succesfully applied template ${TEMPLATE_NAME}"
+echo "Succesfully created table ${TABLE_NAME} from template ${TEMPLATE_NAME}"
diff --git a/pinot-tools/src/main/resources/generator/generator.sh b/pinot-tools/src/main/resources/generator/generator.sh
deleted file mode 100755
index ac1e746..0000000
--- a/pinot-tools/src/main/resources/generator/generator.sh
+++ /dev/null
@@ -1,76 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-
-#ADMIN_PATH="/opt/pinot/bin/pinot-admin.sh"
-ADMIN_PATH="./pinot-tools/target/pinot-tools-pkg/bin/pinot-admin.sh"
-#PATTERN_BASEDIR="/opt/pinot/examples/docker/generators"
-PATTERN_BASEDIR="./pinot-tools/src/main/resources/generator"
-TEMP_DIR="/tmp/pinotGenerator"
-
-if [ -z "$1" ]; then
- echo "No PATTERN name specified. Aborting."
- exit 1
-fi
-
-PATTERN_NAME="$1"
-DATA_DIR="${TEMP_DIR:?}/${PATTERN_NAME}"
-SEGMENT_DIR="${TEMP_DIR:?}/${PATTERN_NAME}Segment"
-
-echo "Preparing temp directory for ${PATTERN_NAME}"
-rm -rf "${DATA_DIR}"
-rm -rf "${SEGMENT_DIR}"
-mkdir -p "${TEMP_DIR}"
-
-echo "Generating data for ${PATTERN_NAME} in ${DATA_DIR}"
-${ADMIN_PATH} GenerateData \
--numFiles 1 -numRecords 354780 -format csv \
--schemaFile "${PATTERN_BASEDIR}/${PATTERN_NAME}_schema.json" \
--schemaAnnotationFile "${PATTERN_BASEDIR}/${PATTERN_NAME}_generator.json" \
--outDir "$DATA_DIR"
-
-if [ ! -d "${DATA_DIR}" ]; then
- echo "Data generation failed. Aborting."
- exit 1
-fi
-
-echo "Creating segment for ${PATTERN_NAME} in ${SEGMENT_DIR}"
-${ADMIN_PATH} CreateSegment \
--tableName "${PATTERN_NAME}" -segmentName "${PATTERN_NAME}" -format CSV -overwrite \
--schemaFile "${PATTERN_BASEDIR}/${PATTERN_NAME}_schema.json" \
--dataDir "${DATA_DIR}" \
--outDir "${SEGMENT_DIR}" || exit 1
-
-if [ ! -d "${SEGMENT_DIR}" ]; then
- echo "Data generation failed. Aborting."
- exit 1
-fi
-
-echo "Adding table ${PATTERN_NAME}"
-${ADMIN_PATH} AddTable -exec \
--tableConfigFile "${PATTERN_BASEDIR}/${PATTERN_NAME}_config.json" \
--schemaFile "${PATTERN_BASEDIR}/${PATTERN_NAME}_schema.json" || exit 1
-
-echo "Uploading segment for ${PATTERN_NAME}"
-${ADMIN_PATH} UploadSegment \
--tableName "${PATTERN_NAME}" \
--segmentDir "${SEGMENT_DIR}" || exit 1
-
-echo "Succesfully applied PATTERN ${PATTERN_NAME}"
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org