You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by mc...@apache.org on 2019/07/21 16:58:48 UTC
[incubator-pinot] branch master updated: Enable
SegmentCompleteionIntegrationTest (#4451)
This is an automated email from the ASF dual-hosted git repository.
mcvsubbu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 024da51 Enable SegmentCompleteionIntegrationTest (#4451)
024da51 is described below
commit 024da51a1ddac76af684761ca25b081c6780cef3
Author: Subbu Subramaniam <mc...@users.noreply.github.com>
AuthorDate: Sun Jul 21 09:58:42 2019 -0700
Enable SegmentCompleteionIntegrationTest (#4451)
* Enable SegmentCompleteionIntegrationTest
This test was not running because of a typo in the class name.
Also, re-organized the test so that it does not have to override
methods from upper level tests each time new tests are added.
* Addressed review comments
---
.../tests/BaseClusterIntegrationTest.java | 34 +++++++++++
.../tests/BaseClusterIntegrationTestSet.java | 11 ----
.../tests/RealtimeClusterIntegrationTest.java | 24 +-------
....java => SegmentCompletionIntegrationTest.java} | 67 +++-------------------
.../perf/BenchmarkRealtimeConsumptionSpeed.java | 2 +-
.../org/apache/pinot/perf/RealtimeStressTest.java | 2 +-
6 files changed, 46 insertions(+), 94 deletions(-)
diff --git a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/BaseClusterIntegrationTest.java b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/BaseClusterIntegrationTest.java
index d94b7b6..70f3d7f 100644
--- a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/BaseClusterIntegrationTest.java
+++ b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/BaseClusterIntegrationTest.java
@@ -20,6 +20,7 @@ package org.apache.pinot.integration.tests;
import com.google.common.base.Function;
import java.io.File;
+import java.io.IOException;
import java.net.URL;
import java.sql.Connection;
import java.sql.DriverManager;
@@ -28,14 +29,18 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.Executor;
+import java.util.concurrent.TimeUnit;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import org.apache.commons.io.FileUtils;
import org.apache.pinot.client.ConnectionFactory;
import org.apache.pinot.common.config.ColumnPartitionConfig;
+import org.apache.pinot.common.config.CombinedConfig;
import org.apache.pinot.common.config.SegmentPartitionConfig;
+import org.apache.pinot.common.config.Serializer;
import org.apache.pinot.common.config.TableTaskConfig;
import org.apache.pinot.common.config.TagNameUtils;
+import org.apache.pinot.common.data.Schema;
import org.apache.pinot.common.utils.TarGzCompressionUtils;
import org.apache.pinot.common.utils.ZkStarter;
import org.apache.pinot.core.realtime.impl.kafka.KafkaConsumerFactory;
@@ -402,4 +407,33 @@ public abstract class BaseClusterIntegrationTest extends ClusterTest {
ClusterIntegrationTestUtils
.testQuery(sqlQuery, "sql", _brokerBaseApiUrl, getPinotConnection(), sqlQueries, getH2Connection());
}
+
+ protected void setUpRealtimeTable(File avroFile)
+ throws Exception {
+ File schemaFile = getSchemaFile();
+ Schema schema = Schema.fromFile(schemaFile);
+ String schemaName = schema.getSchemaName();
+ addSchema(schemaFile, schemaName);
+
+ String timeColumnName = schema.getTimeColumnName();
+ Assert.assertNotNull(timeColumnName);
+ TimeUnit outgoingTimeUnit = schema.getOutgoingTimeUnit();
+ Assert.assertNotNull(outgoingTimeUnit);
+ String timeType = outgoingTimeUnit.toString();
+
+ addRealtimeTable(getTableName(), useLlc(), KafkaStarterUtils.DEFAULT_KAFKA_BROKER, KafkaStarterUtils.DEFAULT_ZK_STR,
+ getKafkaTopic(), getRealtimeSegmentFlushSize(), avroFile, timeColumnName, timeType, schemaName,
+ getBrokerTenant(), getServerTenant(), getLoadMode(), getSortedColumn(),
+ getInvertedIndexColumns(), getBloomFilterIndexColumns(), getRawIndexColumns(), getTaskConfig(),
+ getStreamConsumerFactoryClassName());
+
+ completeTableConfiguration();
+ }
+
+ protected void completeTableConfiguration() throws IOException {
+ if (isUsingNewConfigFormat()) {
+ CombinedConfig combinedConfig = new CombinedConfig(_offlineTableConfig, _realtimeTableConfig, _schema);
+ sendPostRequest(_controllerRequestURLBuilder.forNewTableCreate(), Serializer.serializeToString(combinedConfig));
+ }
+ }
}
diff --git a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/BaseClusterIntegrationTestSet.java b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/BaseClusterIntegrationTestSet.java
index 8dd1b7e..857a16d 100644
--- a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/BaseClusterIntegrationTestSet.java
+++ b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/BaseClusterIntegrationTestSet.java
@@ -492,17 +492,6 @@ public abstract class BaseClusterIntegrationTestSet extends BaseClusterIntegrati
}, 60_000L, errorMessage);
}
- protected void completeTableConfiguration() {
- if (isUsingNewConfigFormat()) {
- CombinedConfig combinedConfig = new CombinedConfig(_offlineTableConfig, _realtimeTableConfig, _schema);
- try {
- sendPostRequest(_controllerRequestURLBuilder.forNewTableCreate(), Serializer.serializeToString(combinedConfig));
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
- }
-
protected void updateTableConfiguration() {
if (isUsingNewConfigFormat()) {
CombinedConfig combinedConfig = new CombinedConfig(_offlineTableConfig, _realtimeTableConfig, _schema);
diff --git a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/RealtimeClusterIntegrationTest.java b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/RealtimeClusterIntegrationTest.java
index 6dc4fc6..4ff1378 100644
--- a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/RealtimeClusterIntegrationTest.java
+++ b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/RealtimeClusterIntegrationTest.java
@@ -71,34 +71,12 @@ public class RealtimeClusterIntegrationTest extends BaseClusterIntegrationTestSe
executor.awaitTermination(10, TimeUnit.MINUTES);
// Create Pinot table
- setUpTable(avroFiles.get(0));
+ setUpRealtimeTable(avroFiles.get(0));
// Wait for all documents loaded
waitForAllDocsLoaded(600_000L);
}
- protected void setUpTable(File avroFile)
- throws Exception {
- File schemaFile = getSchemaFile();
- Schema schema = Schema.fromFile(schemaFile);
- String schemaName = schema.getSchemaName();
- addSchema(schemaFile, schemaName);
-
- String timeColumnName = schema.getTimeColumnName();
- Assert.assertNotNull(timeColumnName);
- TimeUnit outgoingTimeUnit = schema.getOutgoingTimeUnit();
- Assert.assertNotNull(outgoingTimeUnit);
- String timeType = outgoingTimeUnit.toString();
-
- addRealtimeTable(getTableName(), useLlc(), KafkaStarterUtils.DEFAULT_KAFKA_BROKER, KafkaStarterUtils.DEFAULT_ZK_STR,
- getKafkaTopic(), getRealtimeSegmentFlushSize(), avroFile, timeColumnName, timeType, schemaName,
- getBrokerTenant(), getServerTenant(), getLoadMode(), getSortedColumn(),
- getInvertedIndexColumns(), getBloomFilterIndexColumns(), getRawIndexColumns(), getTaskConfig(),
- getStreamConsumerFactoryClassName());
-
- completeTableConfiguration();
- }
-
@Test
@Override
public void testQueriesFromQueryFile()
diff --git a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/SegmentCompletionIntegrationTests.java b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/SegmentCompletionIntegrationTest.java
similarity index 89%
rename from pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/SegmentCompletionIntegrationTests.java
rename to pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/SegmentCompletionIntegrationTest.java
index 7984493..560bf5a 100644
--- a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/SegmentCompletionIntegrationTests.java
+++ b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/SegmentCompletionIntegrationTest.java
@@ -52,7 +52,7 @@ import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
-public class SegmentCompletionIntegrationTests extends LLCRealtimeClusterIntegrationTest {
+public class SegmentCompletionIntegrationTest extends BaseClusterIntegrationTest {
private static final int NUM_KAFKA_PARTITIONS = 1;
private String _serverInstance;
@@ -77,7 +77,12 @@ public class SegmentCompletionIntegrationTests extends LLCRealtimeClusterIntegra
startKafka();
// Create Pinot table
- setUpTable(null);
+ setUpRealtimeTable(null);
+ }
+
+ @Override
+ protected boolean useLlc() {
+ return true;
}
/**
@@ -185,66 +190,12 @@ public class SegmentCompletionIntegrationTests extends LLCRealtimeClusterIntegra
}, 60_000L, "Failed to get a new segment reaching CONSUMING state");
}
- @Test(enabled = false)
- @Override
- public void testQueriesFromQueryFile()
- throws Exception {
- // Skipped
- }
-
- @Test(enabled = false)
- @Override
- public void testSqlQueriesFromQueryFile()
- throws Exception {
- // Skipped
- }
-
- @Test(enabled = false)
- @Override
- public void testGeneratedQueriesWithMultiValues()
- throws Exception {
- // Skipped
- }
-
- @Test(enabled = false)
- @Override
- public void testInstanceShutdown()
- throws Exception {
- // Skipped
- }
-
- @Test(enabled = false)
- @Override
- public void testSegmentFlushSize()
- throws Exception {
- // Skipped
- }
-
- @Test(enabled = false)
- @Override
- public void testDictionaryBasedQueries()
- throws Exception {
- // Skipped
- }
-
- @Test(enabled = false)
- @Override
- public void testQueryExceptions()
- throws Exception {
- // Skipped
- }
-
- @Test(enabled = false)
- public void testConsumerDirectoryExists() {
- // Skipped
- }
-
@AfterClass
- public void tearDown()
- throws Exception {
+ public void tearDown() {
stopFakeServer();
stopBroker();
stopController();
+ stopKafka();
stopZk();
}
diff --git a/pinot-perf/src/main/java/org/apache/pinot/perf/BenchmarkRealtimeConsumptionSpeed.java b/pinot-perf/src/main/java/org/apache/pinot/perf/BenchmarkRealtimeConsumptionSpeed.java
index e623ece..73b0dfe 100644
--- a/pinot-perf/src/main/java/org/apache/pinot/perf/BenchmarkRealtimeConsumptionSpeed.java
+++ b/pinot-perf/src/main/java/org/apache/pinot/perf/BenchmarkRealtimeConsumptionSpeed.java
@@ -82,7 +82,7 @@ public class BenchmarkRealtimeConsumptionSpeed extends RealtimeClusterIntegratio
startServer();
// Create realtime table
- setUpTable(avroFiles.get(0));
+ setUpRealtimeTable(avroFiles.get(0));
// Wait a couple of seconds for all Helix state transitions to happen
Uninterruptibles.sleepUninterruptibly(5, TimeUnit.SECONDS);
diff --git a/pinot-perf/src/main/java/org/apache/pinot/perf/RealtimeStressTest.java b/pinot-perf/src/main/java/org/apache/pinot/perf/RealtimeStressTest.java
index b945fc5..8b00c6b 100644
--- a/pinot-perf/src/main/java/org/apache/pinot/perf/RealtimeStressTest.java
+++ b/pinot-perf/src/main/java/org/apache/pinot/perf/RealtimeStressTest.java
@@ -86,7 +86,7 @@ public class RealtimeStressTest extends RealtimeClusterIntegrationTest {
startServer();
// Create realtime table
- setUpTable(avroFiles.get(0));
+ setUpRealtimeTable(avroFiles.get(0));
// Wait a couple of seconds for all Helix state transitions to happen
Uninterruptibles.sleepUninterruptibly(5, TimeUnit.SECONDS);
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org