You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by xi...@apache.org on 2020/03/07 04:26:25 UTC
[incubator-pinot] branch master updated: Always copy test data out
from jar file (#5119)
This is an automated email from the ASF dual-hosted git repository.
xiangfu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 5096a37 Always copy test data out from jar file (#5119)
5096a37 is described below
commit 5096a3737f554224f78b46636dc9395bca3023df
Author: Xiang Fu <fx...@gmail.com>
AuthorDate: Fri Mar 6 20:26:19 2020 -0800
Always copy test data out from jar file (#5119)
* Always copy test data out from jar file
* Address comments
---
.travis_quickstart.sh | 49 ++++++++++++-
.../org/apache/pinot/tools/HybridQuickstart.java | 3 +
.../tools/admin/command/QuickstartRunner.java | 10 ++-
.../org/apache/pinot/tools/utils/JarUtils.java | 83 ++++++++++++++++++++++
4 files changed, 142 insertions(+), 3 deletions(-)
diff --git a/.travis_quickstart.sh b/.travis_quickstart.sh
index 2158b61..fcfb1b0 100755
--- a/.travis_quickstart.sh
+++ b/.travis_quickstart.sh
@@ -74,7 +74,52 @@ if [ "${COUNT_STAR_RES_3}" -le "${COUNT_STAR_RES_2}" ] || [ "${COUNT_STAR_RES_2}
echo 'Hybrid Quickstart: Not getting incremental counts for 3 consecutive count star queries with 15 seconds interval.'
exit 1
fi
-# kill $PID
-# sleep 30
+kill $PID
+sleep 30
+
+cd bin
+# Test quick-start-batch
+./quick-start-batch.sh &
+PID=$!
+sleep 60
+COUNT_STAR_RES=`curl -X POST --header 'Accept: application/json' -d '{"sql":"select count(*) from baseballStats limit 1","trace":false}' http://localhost:8000/query/sql | jq '.resultTable.rows[0][0]'`
+if [ "${COUNT_STAR_RES}" -ne 97889 ]; then
+ echo 'Batch Quickstart: Incorrect result for count star query.'
+ exit 1
+fi
+kill $PID
+sleep 30
+
+# Test quick-start-streaming
+./quick-start-streaming.sh &
+PID=$!
+sleep 60
+COUNT_STAR_RES_1=`curl -X POST --header 'Accept: application/json' -d '{"sql":"select count(*) from meetupRsvp limit 1","trace":false}' http://localhost:8000/query/sql | jq '.resultTable.rows[0][0]'`
+sleep 15
+COUNT_STAR_RES_2=`curl -X POST --header 'Accept: application/json' -d '{"sql":"select count(*) from meetupRsvp limit 1","trace":false}' http://localhost:8000/query/sql | jq '.resultTable.rows[0][0]'`
+sleep 15
+COUNT_STAR_RES_3=`curl -X POST --header 'Accept: application/json' -d '{"sql":"select count(*) from meetupRsvp limit 1","trace":false}' http://localhost:8000/query/sql | jq '.resultTable.rows[0][0]'`
+if [ "${COUNT_STAR_RES_3}" -le "${COUNT_STAR_RES_2}" ] || [ "${COUNT_STAR_RES_2}" -le "${COUNT_STAR_RES_1}" ]; then
+ echo 'Streaming Quickstart: Not getting incremental counts for 3 consecutive count star queries with 15 seconds interval.'
+ exit 1
+fi
+kill $PID
+sleep 30
+
+# Test quick-start-hybrid
+./quick-start-hybrid.sh &
+PID=$!
+sleep 60
+COUNT_STAR_RES_1=`curl -X POST --header 'Accept: application/json' -d '{"sql":"select count(*) from airlineStats limit 1","trace":false}' http://localhost:8000/query/sql | jq '.resultTable.rows[0][0]'`
+sleep 15
+COUNT_STAR_RES_2=`curl -X POST --header 'Accept: application/json' -d '{"sql":"select count(*) from airlineStats limit 1","trace":false}' http://localhost:8000/query/sql | jq '.resultTable.rows[0][0]'`
+sleep 15
+COUNT_STAR_RES_3=`curl -X POST --header 'Accept: application/json' -d '{"sql":"select count(*) from airlineStats limit 1","trace":false}' http://localhost:8000/query/sql | jq '.resultTable.rows[0][0]'`
+if [ "${COUNT_STAR_RES_3}" -le "${COUNT_STAR_RES_2}" ] || [ "${COUNT_STAR_RES_2}" -le "${COUNT_STAR_RES_1}" ]; then
+ echo 'Hybrid Quickstart: Not getting incremental counts for 3 consecutive count star queries with 15 seconds interval.'
+ exit 1
+fi
+kill $PID
+sleep 30
exit 0
diff --git a/pinot-tools/src/main/java/org/apache/pinot/tools/HybridQuickstart.java b/pinot-tools/src/main/java/org/apache/pinot/tools/HybridQuickstart.java
index 7951a3f..4ab2aeb 100644
--- a/pinot-tools/src/main/java/org/apache/pinot/tools/HybridQuickstart.java
+++ b/pinot-tools/src/main/java/org/apache/pinot/tools/HybridQuickstart.java
@@ -157,6 +157,9 @@ public class HybridQuickstart {
printStatus(Color.YELLOW, "***** 7. Built and pushed an offline segment *****");
printStatus(Color.YELLOW,
"***** 8. Started publishing a Kafka stream for the realtime instance to start consuming *****");
+ printStatus(Color.YELLOW,
+ "***** 9. Sleep 5 Seconds to wait for all components brought up *****");
+ Thread.sleep(5000);
String q1 = "select count(*) from airlineStats limit 10";
printStatus(Color.YELLOW, "Total number of documents in the table");
diff --git a/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/QuickstartRunner.java b/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/QuickstartRunner.java
index bdbbaa6..0858db3 100644
--- a/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/QuickstartRunner.java
+++ b/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/QuickstartRunner.java
@@ -35,6 +35,7 @@ import org.apache.pinot.spi.ingestion.batch.IngestionJobLauncher;
import org.apache.pinot.spi.ingestion.batch.spec.SegmentGenerationJobSpec;
import org.apache.pinot.spi.utils.JsonUtils;
import org.apache.pinot.tools.QuickstartTableRequest;
+import org.apache.pinot.tools.utils.JarUtils;
import org.yaml.snakeyaml.Yaml;
@@ -178,7 +179,14 @@ public class QuickstartRunner {
String inputDirURI = spec.getInputDirURI();
if (!new File(inputDirURI).exists()) {
URL resolvedInputDirURI = QuickstartRunner.class.getClassLoader().getResource(inputDirURI);
- spec.setInputDirURI(resolvedInputDirURI.toURI().toString());
+ if (resolvedInputDirURI.getProtocol().equals("jar")) {
+ String[] splits = resolvedInputDirURI.getFile().split("!");
+ String inputDir = new File(_tempDir, "inputData").toString();
+ JarUtils.copyResourcesToDirectory(splits[0], splits[1].substring(1), inputDir);
+ spec.setInputDirURI(inputDir);
+ } else {
+ spec.setInputDirURI(resolvedInputDirURI.toString());
+ }
}
IngestionJobLauncher.runIngestionJob(spec);
}
diff --git a/pinot-tools/src/main/java/org/apache/pinot/tools/utils/JarUtils.java b/pinot-tools/src/main/java/org/apache/pinot/tools/utils/JarUtils.java
new file mode 100644
index 0000000..b850e1a
--- /dev/null
+++ b/pinot-tools/src/main/java/org/apache/pinot/tools/utils/JarUtils.java
@@ -0,0 +1,83 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.tools.utils;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Enumeration;
+import java.util.jar.JarEntry;
+import java.util.jar.JarFile;
+
+
+public class JarUtils {
+ private static final String JAR_PREFIX = "jar:";
+ private static final String FILE_PREFIX = "file:";
+
+ private JarUtils() {
+ } // non-instantiable
+
+ /**
+ * Copies a directory from a jar file to an external directory.
+ */
+ public static void copyResourcesToDirectory(String fromJarFilePath, String jarDir, String destDir)
+ throws IOException {
+ if (fromJarFilePath.startsWith(JAR_PREFIX)) {
+ fromJarFilePath = fromJarFilePath.substring(JAR_PREFIX.length());
+ }
+ if (fromJarFilePath.startsWith(FILE_PREFIX)) {
+ fromJarFilePath = fromJarFilePath.substring(FILE_PREFIX.length());
+ }
+ JarFile fromJar = new JarFile(fromJarFilePath);
+ for (Enumeration<JarEntry> entries = fromJar.entries(); entries.hasMoreElements(); ) {
+ JarEntry entry = entries.nextElement();
+ if (entry.getName().startsWith(jarDir + "/") && !entry.isDirectory()) {
+ File dest = new File(destDir + "/" + entry.getName().substring(jarDir.length() + 1));
+ File parent = dest.getParentFile();
+ if (parent != null) {
+ parent.mkdirs();
+ }
+
+ FileOutputStream out = new FileOutputStream(dest);
+ InputStream in = fromJar.getInputStream(entry);
+
+ try {
+ byte[] buffer = new byte[8 * 1024];
+
+ int s = 0;
+ while ((s = in.read(buffer)) > 0) {
+ out.write(buffer, 0, s);
+ }
+ } catch (IOException e) {
+ throw new IOException("Could not copy asset from jar file", e);
+ } finally {
+ try {
+ in.close();
+ } catch (IOException ignored) {
+ }
+ try {
+ out.close();
+ } catch (IOException ignored) {
+ }
+ }
+ }
+ }
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org