You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by bs...@apache.org on 2018/12/03 19:57:11 UTC
[4/4] hive git commit: HIVE-20987: Split the MiniDruid Cli tests into
3 different ones eg Druid_only,
Druid_kakfa and kafka_only (Slim B via Jesus Camacho Rodriguez) for each
cluster we are using different ports kafka_only Kafka broker 9093 for
druid_kaf
HIVE-20987: Split the MiniDruid Cli tests into 3 different ones eg Druid_only, Druid_kakfa and kafka_only (Slim B via Jesus Camacho Rodriguez)
for each cluster we are using different ports
kafka_only Kafka broker 9093 for druid_kafka using 9092 druid_only coordinator 9081 broker 9082 hist 9083 druid_kafka using default druid, same thinng is done for derby
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/8e30b5e0
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/8e30b5e0
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/8e30b5e0
Branch: refs/heads/master
Commit: 8e30b5e029570407d8a1db67d322a95db705750e
Parents: 4117a8b
Author: Slim Bouguerra <bs...@apache.org>
Authored: Mon Dec 3 11:54:12 2018 -0800
Committer: Slim Bouguerra <bs...@apache.org>
Committed: Mon Dec 3 11:54:55 2018 -0800
----------------------------------------------------------------------
.../org/apache/hive/druid/ForkingDruidNode.java | 3 -
.../org/apache/hive/druid/MiniDruidCluster.java | 161 +-
.../hive/kafka/SingleNodeKafkaCluster.java | 34 +-
.../hive/cli/TestMiniDruidKafkaCliDriver.java | 63 +
.../hive/cli/TestMiniHiveKafkaCliDriver.java | 63 +
.../test/resources/testconfiguration.properties | 10 +-
.../hadoop/hive/cli/control/CliConfigs.java | 41 +
.../org/apache/hadoop/hive/ql/QTestUtil.java | 532 +++---
.../clientpositive/druidmini_test_insert.q | 2 +-
.../clientpositive/kafka_storage_handler.q | 10 +-
.../druid/druidmini_test_insert.q.out | 4 +-
.../druid/kafka_storage_handler.q.out | 1717 ------------------
.../kafka/kafka_storage_handler.q.out | 1717 ++++++++++++++++++
.../ptest2/conf/deployed/master-mr2.properties | 14 +
.../ptest2/src/main/resources/batch-exec.vm | 2 +-
15 files changed, 2282 insertions(+), 2091 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/8e30b5e0/itests/qtest-druid/src/main/java/org/apache/hive/druid/ForkingDruidNode.java
----------------------------------------------------------------------
diff --git a/itests/qtest-druid/src/main/java/org/apache/hive/druid/ForkingDruidNode.java b/itests/qtest-druid/src/main/java/org/apache/hive/druid/ForkingDruidNode.java
index f81a0ca..8234084 100644
--- a/itests/qtest-druid/src/main/java/org/apache/hive/druid/ForkingDruidNode.java
+++ b/itests/qtest-druid/src/main/java/org/apache/hive/druid/ForkingDruidNode.java
@@ -27,13 +27,10 @@ import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
-import java.net.URLClassLoader;
-import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;
-import java.util.stream.Collectors;
public class ForkingDruidNode extends DruidNode {
private final static String DEFAULT_JAVA_CMD = "java";
http://git-wip-us.apache.org/repos/asf/hive/blob/8e30b5e0/itests/qtest-druid/src/main/java/org/apache/hive/druid/MiniDruidCluster.java
----------------------------------------------------------------------
diff --git a/itests/qtest-druid/src/main/java/org/apache/hive/druid/MiniDruidCluster.java b/itests/qtest-druid/src/main/java/org/apache/hive/druid/MiniDruidCluster.java
index a9d381f..8081595 100644
--- a/itests/qtest-druid/src/main/java/org/apache/hive/druid/MiniDruidCluster.java
+++ b/itests/qtest-druid/src/main/java/org/apache/hive/druid/MiniDruidCluster.java
@@ -31,6 +31,7 @@ import java.net.DatagramSocket;
import java.net.ServerSocket;
import java.util.Arrays;
import java.util.List;
+import java.util.Locale;
import java.util.Map;
/**
@@ -39,38 +40,48 @@ import java.util.Map;
public class MiniDruidCluster extends AbstractService {
private static final Logger log = LoggerFactory.getLogger(MiniDruidCluster.class);
- private static final String COMMON_DRUID_JVM_PROPPERTIES = "-Duser.timezone=UTC -Dfile.encoding=UTF-8 -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager -Ddruid.emitter=logging -Ddruid.emitter.logging.logLevel=info";
-
- private static final List<String> HISTORICAL_JVM_CONF = Arrays
- .asList("-server", "-XX:MaxDirectMemorySize=10g", "-Xmx512m", "-Xmx512m",
- COMMON_DRUID_JVM_PROPPERTIES
- );
-
- private static final List<String> COORDINATOR_JVM_CONF = Arrays
- .asList("-server", "-XX:MaxDirectMemorySize=2g", "-Xmx512m", "-Xms512m",
- COMMON_DRUID_JVM_PROPPERTIES
- );
-
- private static final Map<String, String> COMMON_DRUID_CONF = ImmutableMap.of(
- "druid.metadata.storage.type", "derby",
- "druid.storage.type", "hdfs",
- "druid.processing.buffer.sizeBytes", "213870912",
- "druid.processing.numThreads", "2",
- "druid.worker.capacity", "4"
- );
-
- private static final Map<String, String> COMMON_DRUID_HISTORICAL = ImmutableMap.of(
- "druid.server.maxSize", "130000000000"
- );
-
- private static final Map<String, String> COMMON_COORDINATOR_INDEXER = ImmutableMap
- .of(
- "druid.indexer.logs.type", "file",
- "druid.coordinator.asOverlord.enabled", "true",
- "druid.coordinator.asOverlord.overlordService", "druid/overlord",
- "druid.coordinator.period", "PT2S",
- "druid.manager.segments.pollDuration", "PT2S"
- );
+ private static final String
+ COMMON_DRUID_JVM_PROPERTIES =
+ "-Duser.timezone=UTC -Dfile.encoding=UTF-8 -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager "
+ + "-Ddruid.emitter=logging -Ddruid.emitter.logging.logLevel=info";
+
+ private static final List<String>
+ HISTORICAL_JVM_CONF =
+ Arrays.asList("-server", "-XX:MaxDirectMemorySize=10g", "-Xmx512m", "-Xmx512m", COMMON_DRUID_JVM_PROPERTIES);
+
+ private static final List<String>
+ COORDINATOR_JVM_CONF =
+ Arrays.asList("-server", "-XX:MaxDirectMemorySize=2g", "-Xmx512m", "-Xms512m", COMMON_DRUID_JVM_PROPERTIES);
+
+ private static final Map<String, String>
+ COMMON_DRUID_CONF =
+ ImmutableMap.of("druid.metadata.storage.type",
+ "derby",
+ "druid.storage.type",
+ "hdfs",
+ "druid.processing.buffer.sizeBytes",
+ "213870912",
+ "druid.processing.numThreads",
+ "2",
+ "druid.worker.capacity",
+ "4");
+
+ private static final Map<String, String>
+ COMMON_DRUID_HISTORICAL =
+ ImmutableMap.of("druid.server.maxSize", "130000000000");
+
+ private static final Map<String, String>
+ COMMON_COORDINATOR_INDEXER =
+ ImmutableMap.of("druid.indexer.logs.type",
+ "file",
+ "druid.coordinator.asOverlord.enabled",
+ "true",
+ "druid.coordinator.asOverlord.overlordService",
+ "druid/overlord",
+ "druid.coordinator.period",
+ "PT2S",
+ "druid.manager.segments.pollDuration",
+ "PT2S");
private static final int MIN_PORT_NUMBER = 60000;
private static final int MAX_PORT_NUMBER = 65535;
@@ -87,36 +98,41 @@ public class MiniDruidCluster extends AbstractService {
private final File logDirectory;
private final String derbyURI;
-
- public MiniDruidCluster(String name) {
- this(name, "/tmp/miniDruid/log", "/tmp/miniDruid/data", 2181, null);
- }
+ private final int coordinatorPort;
+ private final int historicalPort;
+ private final int brokerPort;
public MiniDruidCluster(String name, String logDir, String tmpDir, Integer zookeeperPort, String classpath) {
super(name);
this.dataDirectory = new File(tmpDir, "druid-data");
this.logDirectory = new File(logDir);
- int derbyPort = findPort(MIN_PORT_NUMBER, MAX_PORT_NUMBER);
+ boolean isKafka = name.contains("kafka");
+ coordinatorPort = isKafka ? 8081 : 9081;
+ brokerPort = coordinatorPort + 1;
+ historicalPort = brokerPort + 1;
+ int derbyPort = historicalPort + 1;
ensureCleanDirectory(dataDirectory);
- derbyURI = String
- .format("jdbc:derby://localhost:%s/%s/druid_derby/metadata.db;create=true",
- derbyPort,
- dataDirectory.getAbsolutePath()
- );
- String segmentsCache = String
- .format("[{\"path\":\"%s/druid/segment-cache\",\"maxSize\":130000000000}]",
- dataDirectory.getAbsolutePath()
- );
+ derbyURI =
+ String.format("jdbc:derby://localhost:%s/%s/druid_derby/metadata.db;create=true",
+ derbyPort,
+ dataDirectory.getAbsolutePath());
+ String
+ segmentsCache =
+ String.format("[{\"path\":\"%s/druid/segment-cache\",\"maxSize\":130000000000}]",
+ dataDirectory.getAbsolutePath());
String indexingLogDir = new File(logDirectory, "indexer-log").getAbsolutePath();
ImmutableMap.Builder<String, String> coordinatorMapBuilder = new ImmutableMap.Builder();
ImmutableMap.Builder<String, String> historicalMapBuilder = new ImmutableMap.Builder();
+ ImmutableMap.Builder<String, String> brokerMapBuilder = new ImmutableMap.Builder();
- Map<String, String> coordinatorProperties = coordinatorMapBuilder.putAll(COMMON_DRUID_CONF)
+ Map<String, String>
+ coordinatorProperties =
+ coordinatorMapBuilder.putAll(COMMON_DRUID_CONF)
.putAll(COMMON_COORDINATOR_INDEXER)
.put("druid.metadata.storage.connector.connectURI", derbyURI)
.put("druid.metadata.storage.connector.port", String.valueOf(derbyPort))
@@ -125,23 +141,28 @@ public class MiniDruidCluster extends AbstractService {
.put("druid.coordinator.startDelay", "PT1S")
.put("druid.indexer.runner", "local")
.put("druid.storage.storageDirectory", getDeepStorageDir())
+ .put("druid.port", String.valueOf(coordinatorPort))
.build();
- Map<String, String> historicalProperties = historicalMapBuilder.putAll(COMMON_DRUID_CONF)
+ Map<String, String>
+ historicalProperties =
+ historicalMapBuilder.putAll(COMMON_DRUID_CONF)
.putAll(COMMON_DRUID_HISTORICAL)
.put("druid.zk.service.host", "localhost:" + zookeeperPort)
.put("druid.segmentCache.locations", segmentsCache)
.put("druid.storage.storageDirectory", getDeepStorageDir())
+ .put("druid.port", String.valueOf(historicalPort))
+ .build();
+ Map<String, String>
+ brokerProperties =
+ brokerMapBuilder.putAll(COMMON_DRUID_CONF)
+ .put("druid.zk.service.host", "localhost:" + zookeeperPort)
+ .put("druid.port", String.valueOf(brokerPort))
.build();
- coordinator = new ForkingDruidNode("coordinator", classpath, coordinatorProperties,
- COORDINATOR_JVM_CONF,
- logDirectory, null
- );
- historical = new ForkingDruidNode("historical", classpath, historicalProperties, HISTORICAL_JVM_CONF,
- logDirectory, null
- );
- broker = new ForkingDruidNode("broker", classpath, historicalProperties, HISTORICAL_JVM_CONF,
- logDirectory, null
- );
+ coordinator =
+ new ForkingDruidNode("coordinator", classpath, coordinatorProperties, COORDINATOR_JVM_CONF, logDirectory, null);
+ historical =
+ new ForkingDruidNode("historical", classpath, historicalProperties, HISTORICAL_JVM_CONF, logDirectory, null);
+ broker = new ForkingDruidNode("broker", classpath, brokerProperties, HISTORICAL_JVM_CONF, logDirectory, null);
druidNodes = Arrays.asList(coordinator, historical, broker);
}
@@ -151,7 +172,7 @@ public class MiniDruidCluster extends AbstractService {
while (!available(port)) {
port++;
if (port == end) {
- throw new RuntimeException("can not find free port for range " + start + ":" + end);
+ throw new RuntimeException("can not find free port for range " + start + ":" + end);
}
}
return port;
@@ -193,7 +214,7 @@ public class MiniDruidCluster extends AbstractService {
return false;
}
- private static void ensureCleanDirectory(File dir){
+ public static void ensureCleanDirectory(File dir) {
try {
if (dir.exists()) {
// need to clean data directory to ensure that there is no interference from old runs
@@ -211,14 +232,12 @@ public class MiniDruidCluster extends AbstractService {
}
}
- @Override
- protected void serviceStart() throws Exception {
+ @Override protected void serviceStart() throws Exception {
druidNodes.stream().forEach(node -> {
try {
node.start();
} catch (IOException e) {
- log.error("Failed to start node " + node.getNodeType()
- + " Consequently will destroy the cluster");
+ log.error("Failed to start node " + node.getNodeType() + " Consequently will destroy the cluster");
druidNodes.stream().filter(node1 -> node1.isAlive()).forEach(nodeToStop -> {
try {
log.info("Stopping Node " + nodeToStop.getNodeType());
@@ -232,8 +251,7 @@ public class MiniDruidCluster extends AbstractService {
});
}
- @Override
- protected void serviceStop() throws Exception {
+ @Override protected void serviceStop() throws Exception {
druidNodes.stream().forEach(node -> {
try {
node.close();
@@ -244,7 +262,6 @@ public class MiniDruidCluster extends AbstractService {
});
}
-
public String getMetadataURI() {
return derbyURI;
}
@@ -253,11 +270,15 @@ public class MiniDruidCluster extends AbstractService {
return dataDirectory.getAbsolutePath() + File.separator + "deep-storage";
}
- public String getCoordinatorURI(){
- return "localhost:8081";
+ public String getCoordinatorURI() {
+ return String.format(Locale.ROOT,"localhost:%s", coordinatorPort);
+ }
+
+ public String getBrokerURI() {
+ return String.format(Locale.ROOT,"localhost:%s", brokerPort);
}
- public String getOverlordURI(){
+ public String getOverlordURI() {
// Overlord and coordinator both run in same JVM.
return getCoordinatorURI();
}
http://git-wip-us.apache.org/repos/asf/hive/blob/8e30b5e0/itests/qtest-druid/src/main/java/org/apache/hive/kafka/SingleNodeKafkaCluster.java
----------------------------------------------------------------------
diff --git a/itests/qtest-druid/src/main/java/org/apache/hive/kafka/SingleNodeKafkaCluster.java b/itests/qtest-druid/src/main/java/org/apache/hive/kafka/SingleNodeKafkaCluster.java
index 3f2c9a7..348bc78 100644
--- a/itests/qtest-druid/src/main/java/org/apache/hive/kafka/SingleNodeKafkaCluster.java
+++ b/itests/qtest-druid/src/main/java/org/apache/hive/kafka/SingleNodeKafkaCluster.java
@@ -7,7 +7,9 @@ import kafka.server.KafkaServerStartable;
import kafka.utils.ZKStringSerializer$;
import kafka.utils.ZkUtils;
+import org.apache.commons.io.FileUtils;
import org.apache.hadoop.service.AbstractService;
+import org.apache.hive.druid.MiniDruidCluster;
import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.ProducerRecord;
import org.apache.kafka.clients.producer.RecordMetadata;
@@ -35,26 +37,44 @@ import java.util.stream.IntStream;
/**
* This class has the hooks to start and stop single node kafka cluster.
- * The kafka broker is started on port 9092
+ *
*/
public class SingleNodeKafkaCluster extends AbstractService {
private static final Logger log = LoggerFactory.getLogger(SingleNodeKafkaCluster.class);
private static final int BROKER_PORT = 9092;
private static final String LOCALHOST = "localhost";
- private static final String LOCALHOST_9092 = String.format("%s:%s", LOCALHOST, BROKER_PORT);
private final KafkaServerStartable serverStartable;
private final String zkString;
+ private final int brokerPort;
+ private final String kafkaServer;
- public SingleNodeKafkaCluster(String name, String logDir, Integer zkPort){
+ public SingleNodeKafkaCluster(String name, String logDir, Integer zkPort, Integer brokerPort){
super(name);
Properties properties = new Properties();
+ this.brokerPort = brokerPort == null ? BROKER_PORT : brokerPort;
+ File dir = new File(logDir);
+ if (dir.exists()) {
+ // need to clean data directory to ensure that there is no interference from old runs
+ // Cleaning is happening here to allow debugging in case of tests fail
+ // we don;t have to clean logs since it is an append mode
+ log.info("Cleaning the druid directory [{}]", dir.getAbsolutePath());
+ try {
+ FileUtils.deleteDirectory(dir);
+ } catch (IOException e) {
+ log.error("Failed to clean druid directory");
+ throw new RuntimeException(e);
+ }
+ }
this.zkString = String.format("localhost:%d", zkPort);
+
+ this.kafkaServer = String.format("%s:%d", LOCALHOST, this.brokerPort);
+
properties.setProperty("zookeeper.connect", zkString);
properties.setProperty("broker.id", String.valueOf(1));
properties.setProperty("host.name", LOCALHOST);
- properties.setProperty("port", Integer.toString(BROKER_PORT));
+ properties.setProperty("port", Integer.toString(brokerPort));
properties.setProperty("log.dir", logDir);
// This property is very important, we are sending form records with a specific time
// Thus need to make sure that they don't get DELETED
@@ -73,7 +93,7 @@ public class SingleNodeKafkaCluster extends AbstractService {
@Override
protected void serviceStart() throws Exception {
serverStartable.startup();
- log.info("Kafka Server Started");
+ log.info("Kafka Server Started on port {}", brokerPort);
}
@@ -94,7 +114,7 @@ public class SingleNodeKafkaCluster extends AbstractService {
createTopic(topicName);
// set up kafka producer
Properties properties = new Properties();
- properties.put("bootstrap.servers", LOCALHOST_9092);
+ properties.put("bootstrap.servers", kafkaServer);
properties.put("acks", "1");
properties.put("retries", "3");
@@ -116,7 +136,7 @@ public class SingleNodeKafkaCluster extends AbstractService {
createTopic(topic);
// set up kafka producer
Properties properties = new Properties();
- properties.put("bootstrap.servers", LOCALHOST_9092);
+ properties.put("bootstrap.servers", kafkaServer);
properties.put("acks", "1");
properties.put("retries", "3");
http://git-wip-us.apache.org/repos/asf/hive/blob/8e30b5e0/itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestMiniDruidKafkaCliDriver.java
----------------------------------------------------------------------
diff --git a/itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestMiniDruidKafkaCliDriver.java b/itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestMiniDruidKafkaCliDriver.java
new file mode 100644
index 0000000..a330aff
--- /dev/null
+++ b/itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestMiniDruidKafkaCliDriver.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.cli;
+
+import org.apache.hadoop.hive.cli.control.CliAdapter;
+import org.apache.hadoop.hive.cli.control.CliConfigs;
+import org.junit.ClassRule;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TestRule;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
+
+import java.io.File;
+import java.util.List;
+
+@RunWith(Parameterized.class)
+public class TestMiniDruidKafkaCliDriver
+{
+
+ static CliAdapter adapter = new CliConfigs.MiniDruidKafkaCliConfig().getCliAdapter();
+
+ @Parameters(name = "{0}")
+ public static List<Object[]> getParameters() throws Exception {
+ return adapter.getParameters();
+ }
+
+ @ClassRule
+ public static TestRule cliClassRule = adapter.buildClassRule();
+
+ @Rule
+ public TestRule cliTestRule = adapter.buildTestRule();
+
+ private String name;
+ private File qfile;
+
+ public TestMiniDruidKafkaCliDriver(String name, File qfile) {
+ this.name = name;
+ this.qfile = qfile;
+ }
+
+ @Test
+ public void testCliDriver() throws Exception {
+ adapter.runTest(name, qfile);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/8e30b5e0/itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestMiniHiveKafkaCliDriver.java
----------------------------------------------------------------------
diff --git a/itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestMiniHiveKafkaCliDriver.java b/itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestMiniHiveKafkaCliDriver.java
new file mode 100644
index 0000000..d1eeb55
--- /dev/null
+++ b/itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestMiniHiveKafkaCliDriver.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.cli;
+
+import org.apache.hadoop.hive.cli.control.CliAdapter;
+import org.apache.hadoop.hive.cli.control.CliConfigs;
+import org.junit.ClassRule;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TestRule;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
+
+import java.io.File;
+import java.util.List;
+
+@RunWith(Parameterized.class)
+public class TestMiniHiveKafkaCliDriver
+{
+
+ static CliAdapter adapter = new CliConfigs.MiniKafkaCliConfig().getCliAdapter();
+
+ @Parameters(name = "{0}")
+ public static List<Object[]> getParameters() throws Exception {
+ return adapter.getParameters();
+ }
+
+ @ClassRule
+ public static TestRule cliClassRule = adapter.buildClassRule();
+
+ @Rule
+ public TestRule cliTestRule = adapter.buildTestRule();
+
+ private String name;
+ private File qfile;
+
+ public TestMiniHiveKafkaCliDriver(String name, File qfile) {
+ this.name = name;
+ this.qfile = qfile;
+ }
+
+ @Test
+ public void testCliDriver() throws Exception {
+ adapter.runTest(name, qfile);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/8e30b5e0/itests/src/test/resources/testconfiguration.properties
----------------------------------------------------------------------
diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties
index 5aadf2c..d8f559c 100644
--- a/itests/src/test/resources/testconfiguration.properties
+++ b/itests/src/test/resources/testconfiguration.properties
@@ -1848,12 +1848,14 @@ druid.query.files=druidmini_test1.q,\
druidmini_extractTime.q,\
druidmini_test_alter.q,\
druidmini_floorTime.q, \
- druidmini_masking.q, \
- druidkafkamini_basic.q, \
+ druidmini_masking.q
+
+druid.kafka.query.files=druidkafkamini_basic.q, \
druidkafkamini_avro.q, \
druidkafkamini_csv.q, \
- druidkafkamini_delimited.q, \
- kafka_storage_handler.q
+ druidkafkamini_delimited.q
+
+hive.kafka.query.files=kafka_storage_handler.q
druid.llap.local.query.files=druidmini_noop.q
http://git-wip-us.apache.org/repos/asf/hive/blob/8e30b5e0/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java
index 7ac7ba1..2017c94 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java
@@ -56,6 +56,8 @@ public class CliConfigs {
excludesFrom(testConfigProps, "disabled.query.files");
excludesFrom(testConfigProps, "localSpark.only.query.files");
excludesFrom(testConfigProps, "druid.query.files");
+ excludesFrom(testConfigProps, "druid.kafka.query.files");
+ excludesFrom(testConfigProps, "hive.kafka.query.files");
excludesFrom(testConfigProps, "erasurecoding.only.query.files");
excludeQuery("fouter_join_ppr.q"); // Disabled in HIVE-19509
@@ -190,6 +192,27 @@ public class CliConfigs {
setInitScript("q_test_druid_init.sql");
setCleanupScript("q_test_cleanup_druid.sql");
setHiveConfDir("data/conf/llap");
+ setClusterType(MiniClusterType.druid);
+ setMetastoreType(MetastoreType.sql);
+ setFsType(QTestUtil.FsType.hdfs);
+ } catch (Exception e) {
+ throw new RuntimeException("can't construct cliconfig", e);
+ }
+ }
+ }
+
+ public static class MiniDruidKafkaCliConfig extends AbstractCliConfig {
+ public MiniDruidKafkaCliConfig() {
+ super(CoreCliDriver.class);
+ try {
+ setQueryDir("ql/src/test/queries/clientpositive");
+ includesFrom(testConfigProps, "druid.kafka.query.files");
+ setResultsDir("ql/src/test/results/clientpositive/druid");
+ setLogDir("itests/qtest/target/tmp/log");
+
+ setInitScript("q_test_druid_init.sql");
+ setCleanupScript("q_test_cleanup_druid.sql");
+ setHiveConfDir("data/conf/llap");
setClusterType(MiniClusterType.druidKafka);
setMetastoreType(MetastoreType.sql);
setFsType(QTestUtil.FsType.hdfs);
@@ -199,6 +222,24 @@ public class CliConfigs {
}
}
+ public static class MiniKafkaCliConfig extends AbstractCliConfig {
+ public MiniKafkaCliConfig() {
+ super(CoreCliDriver.class);
+ try {
+ setQueryDir("ql/src/test/queries/clientpositive");
+ includesFrom(testConfigProps, "hive.kafka.query.files");
+ setResultsDir("ql/src/test/results/clientpositive/kafka");
+ setLogDir("itests/qtest/target/tmp/log");
+ setHiveConfDir("data/conf/llap");
+ setClusterType(MiniClusterType.kafka);
+ setMetastoreType(MetastoreType.sql);
+ setFsType(QTestUtil.FsType.hdfs);
+ } catch (Exception e) {
+ throw new RuntimeException("can't construct cliconfig", e);
+ }
+ }
+ }
+
public static class MiniLlapLocalCliConfig extends AbstractCliConfig {
public MiniLlapLocalCliConfig() {
http://git-wip-us.apache.org/repos/asf/hive/blob/8e30b5e0/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
index dcf2d49..886f85e 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
@@ -135,7 +135,6 @@ import com.google.common.collect.ImmutableList;
/**
* QTestUtil.
- *
*/
public class QTestUtil {
@@ -150,21 +149,19 @@ public class QTestUtil {
private final static String defaultInitScript = "q_test_init.sql";
private final static String defaultCleanupScript = "q_test_cleanup.sql";
private static SimpleDateFormat formatter = new SimpleDateFormat("MM/dd/yyyy HH:mm:ss");
- private final String[] testOnlyCommands = new String[]{"crypto", "erasure"};
+ private final String[] testOnlyCommands = new String[]{ "crypto", "erasure" };
public static final String TEST_TMP_DIR_PROPERTY = "test.tmp.dir"; // typically target/tmp
private static final String BUILD_DIR_PROPERTY = "build.dir"; // typically target
private static final String TEST_SRC_TABLES_PROPERTY = "test.src.tables";
-
/**
* The default Erasure Coding Policy to use in Erasure Coding tests.
*/
public static final String DEFAULT_TEST_EC_POLICY = "RS-3-2-1024k";
private String testWarehouse;
- @Deprecated
- private final String testFiles;
+ @Deprecated private final String testFiles;
private final File datasetDir;
private final String outDir;
protected final String logDir;
@@ -204,19 +201,19 @@ public class QTestUtil {
private SingleNodeKafkaCluster kafkaCluster;
public static Set<String> getSrcTables() {
- if (srcTables == null){
+ if (srcTables == null) {
initSrcTables();
}
return srcTables;
}
- public static void addSrcTable(String table){
+ public static void addSrcTable(String table) {
getSrcTables().add(table);
storeSrcTables();
}
public static Set<String> initSrcTables() {
- if (srcTables == null){
+ if (srcTables == null) {
initSrcTablesFromSystemProperty();
storeSrcTables();
}
@@ -228,7 +225,7 @@ public class QTestUtil {
System.setProperty(TEST_SRC_TABLES_PROPERTY, String.join(",", srcTables));
}
- private static void initSrcTablesFromSystemProperty(){
+ private static void initSrcTablesFromSystemProperty() {
srcTables = new HashSet<String>();
// FIXME: moved default value to here...for now
// i think this features is never really used from the command line
@@ -241,7 +238,7 @@ public class QTestUtil {
}
private CliDriver getCliDriver() {
- if(cliDriver == null){
+ if (cliDriver == null) {
throw new RuntimeException("no clidriver");
}
return cliDriver;
@@ -249,6 +246,7 @@ public class QTestUtil {
/**
* Returns the default UDF names which should not be removed when resetting the test database
+ *
* @return The list of the UDF names not to remove
*/
private Set<String> getSrcUDFs() {
@@ -256,8 +254,7 @@ public class QTestUtil {
// FIXME: moved default value to here...for now
// i think this features is never really used from the command line
String defaultTestSrcUDFs = "qtest_get_java_boolean";
- for (String srcUDF : System.getProperty("test.src.udfs", defaultTestSrcUDFs).trim().split(","))
- {
+ for (String srcUDF : System.getProperty("test.src.udfs", defaultTestSrcUDFs).trim().split(",")) {
srcUDF = srcUDF.trim();
if (!srcUDF.isEmpty()) {
srcUDFs.add(srcUDF);
@@ -276,19 +273,18 @@ public class QTestUtil {
public void initConf() throws Exception {
String vectorizationEnabled = System.getProperty("test.vectorization.enabled");
- if(vectorizationEnabled != null && vectorizationEnabled.equalsIgnoreCase("true")) {
+ if (vectorizationEnabled != null && vectorizationEnabled.equalsIgnoreCase("true")) {
conf.setBoolVar(ConfVars.HIVE_VECTORIZATION_ENABLED, true);
}
// Plug verifying metastore in for testing DirectSQL.
- conf.setVar(ConfVars.METASTORE_RAW_STORE_IMPL,
- "org.apache.hadoop.hive.metastore.VerifyingObjectStore");
+ conf.setVar(ConfVars.METASTORE_RAW_STORE_IMPL, "org.apache.hadoop.hive.metastore.VerifyingObjectStore");
if (mr != null) {
mr.setupConfiguration(conf);
// TODO Ideally this should be done independent of whether mr is setup or not.
- setFsRelatedProperties(conf, fs.getScheme().equals("file"),fs);
+ setFsRelatedProperties(conf, fs.getScheme().equals("file"), fs);
}
if (llapCluster != null) {
@@ -306,8 +302,8 @@ public class QTestUtil {
conf.set("hive.druid.metadata.uri", druidCluster.getMetadataURI());
conf.set("hive.druid.coordinator.address.default", druidCluster.getCoordinatorURI());
conf.set("hive.druid.overlord.address.default", druidCluster.getOverlordURI());
- final Path scratchDir = fs
- .makeQualified(new Path(System.getProperty("test.tmp.dir"), "druidStagingDir"));
+ conf.set("hive.druid.broker.address.default", druidCluster.getBrokerURI());
+ final Path scratchDir = fs.makeQualified(new Path(System.getProperty("test.tmp.dir"), "druidStagingDir"));
fs.mkdirs(scratchDir);
conf.set("hive.druid.working.directory", scratchDir.toUri().getPath());
}
@@ -326,7 +322,7 @@ public class QTestUtil {
Path path = new Path(fsUriString, buildDir);
// Create a fake fs root for local fs
- Path localFsRoot = new Path(path, "localfs");
+ Path localFsRoot = new Path(path, "localfs");
warehousePath = new Path(localFsRoot, "warehouse");
jarPath = new Path(localFsRoot, "jar");
userInstallPath = new Path(localFsRoot, "user_install");
@@ -366,50 +362,36 @@ public class QTestUtil {
try {
fs.mkdirs(warehousePath);
} catch (IOException e) {
- LOG.error("Failed to create path={}. Continuing. Exception message={}", warehousePath,
- e.getMessage());
+ LOG.error("Failed to create path={}. Continuing. Exception message={}", warehousePath, e.getMessage());
}
try {
fs.mkdirs(hiveJarPath);
} catch (IOException e) {
- LOG.error("Failed to create path={}. Continuing. Exception message={}", warehousePath,
- e.getMessage());
+ LOG.error("Failed to create path={}. Continuing. Exception message={}", warehousePath, e.getMessage());
}
try {
fs.mkdirs(userInstallPath);
} catch (IOException e) {
- LOG.error("Failed to create path={}. Continuing. Exception message={}", warehousePath,
- e.getMessage());
+ LOG.error("Failed to create path={}. Continuing. Exception message={}", warehousePath, e.getMessage());
}
}
private enum CoreClusterType {
- MR,
- TEZ,
- SPARK
+ MR, TEZ, SPARK
}
public enum FsType {
- local,
- hdfs,
- encrypted_hdfs,
- erasure_coded_hdfs,
+ local, hdfs, encrypted_hdfs, erasure_coded_hdfs,
}
public enum MiniClusterType {
- mr(CoreClusterType.MR, FsType.hdfs),
- tez(CoreClusterType.TEZ, FsType.hdfs),
- tez_local(CoreClusterType.TEZ, FsType.local),
- spark(CoreClusterType.SPARK, FsType.local),
- miniSparkOnYarn(CoreClusterType.SPARK, FsType.hdfs),
- llap(CoreClusterType.TEZ, FsType.hdfs),
- llap_local(CoreClusterType.TEZ, FsType.local),
- none(CoreClusterType.MR, FsType.local),
- druidLocal(CoreClusterType.TEZ, FsType.local),
- druidKafka(CoreClusterType.TEZ, FsType.hdfs),
- kafka(CoreClusterType.TEZ, FsType.hdfs);
-
+ mr(CoreClusterType.MR, FsType.hdfs), tez(CoreClusterType.TEZ, FsType.hdfs), tez_local(CoreClusterType.TEZ,
+ FsType.local), spark(CoreClusterType.SPARK, FsType.local), miniSparkOnYarn(CoreClusterType.SPARK,
+ FsType.hdfs), llap(CoreClusterType.TEZ, FsType.hdfs), llap_local(CoreClusterType.TEZ, FsType.local), none(
+ CoreClusterType.MR,
+ FsType.local), druidLocal(CoreClusterType.TEZ, FsType.local), druid(CoreClusterType.TEZ,
+ FsType.hdfs), druidKafka(CoreClusterType.TEZ, FsType.hdfs), kafka(CoreClusterType.TEZ, FsType.hdfs);
private final CoreClusterType coreClusterType;
private final FsType defaultFsType;
@@ -445,10 +427,13 @@ public class QTestUtil {
return llap_local;
} else if (type.equals("druidLocal")) {
return druidLocal;
+ } else if (type.equals("druid")) {
+ return druid;
} else if (type.equals("druid-kafka")) {
return druidKafka;
- }
- else {
+ } else if (type.equals("kafka")) {
+ return kafka;
+ } else {
return none;
}
}
@@ -464,10 +449,16 @@ public class QTestUtil {
}
public QTestUtil(QTestArguments testArgs) throws Exception {
- LOG.info("Setting up QTestUtil with outDir={}, logDir={}, clusterType={}, confDir={}," +
- " initScript={}, cleanupScript={}, withLlapIo={}, fsType={}",
- testArgs.getOutDir(), testArgs.getLogDir(), testArgs.getClusterType(), testArgs.getConfDir(),
- testArgs.getInitScript(), testArgs.getCleanupScript(), testArgs.isWithLlapIo(), testArgs.getFsType());
+ LOG.info("Setting up QTestUtil with outDir={}, logDir={}, clusterType={}, confDir={},"
+ + " initScript={}, cleanupScript={}, withLlapIo={}, fsType={}",
+ testArgs.getOutDir(),
+ testArgs.getLogDir(),
+ testArgs.getClusterType(),
+ testArgs.getConfDir(),
+ testArgs.getInitScript(),
+ testArgs.getCleanupScript(),
+ testArgs.isWithLlapIo(),
+ testArgs.getFsType());
Preconditions.checkNotNull(testArgs.getClusterType(), "ClusterType cannot be null");
@@ -479,10 +470,11 @@ public class QTestUtil {
// HIVE-14443 move this fall-back logic to CliConfigs
if (testArgs.getConfDir() != null && !testArgs.getConfDir().isEmpty()) {
- HiveConf.setHiveSiteLocation(new URL(
- "file://"+ new File(testArgs.getConfDir()).toURI().getPath() + "/hive-site.xml"));
+ HiveConf.setHiveSiteLocation(new URL("file://"
+ + new File(testArgs.getConfDir()).toURI().getPath()
+ + "/hive-site.xml"));
MetastoreConf.setHiveSiteLocation(HiveConf.getHiveSiteLocation());
- System.out.println("Setting hive-site: "+HiveConf.getHiveSiteLocation());
+ System.out.println("Setting hive-site: " + HiveConf.getHiveSiteLocation());
}
queryState = new QueryState.Builder().withHiveConf(new HiveConf(IDriver.class)).build();
@@ -510,7 +502,6 @@ public class QTestUtil {
LlapProxy.initializeLlapIo(conf);
}
-
// Use the current directory if it is not specified
String dataDir = conf.get("test.data.files");
if (dataDir == null) {
@@ -520,9 +511,10 @@ public class QTestUtil {
conf.set("test.data.dir", dataDir);
// Use path relative to dataDir directory if it is not specified
- datasetDir = conf.get("test.data.set.files") == null
- ? new File(new File(dataDir).getAbsolutePath() + "/datasets")
- : new File(conf.get("test.data.set.files"));
+ datasetDir =
+ conf.get("test.data.set.files") == null ?
+ new File(new File(dataDir).getAbsolutePath() + "/datasets") :
+ new File(conf.get("test.data.set.files"));
String scriptsDir = getScriptsDir();
@@ -552,7 +544,7 @@ public class QTestUtil {
if (fsType == FsType.local) {
fs = FileSystem.getLocal(conf);
- } else if (fsType == FsType.hdfs || fsType == FsType.encrypted_hdfs|| fsType == FsType.erasure_coded_hdfs) {
+ } else if (fsType == FsType.hdfs || fsType == FsType.encrypted_hdfs || fsType == FsType.erasure_coded_hdfs) {
int numDataNodes = 4;
// Setup before getting dfs
@@ -599,80 +591,70 @@ public class QTestUtil {
}
}
- private void setupMiniCluster(HadoopShims shims, String confDir) throws
- IOException {
+ private void setupMiniCluster(HadoopShims shims, String confDir) throws IOException {
String uriString = fs.getUri().toString();
if (clusterType == MiniClusterType.druidKafka
- || clusterType == MiniClusterType.druidLocal) {
+ || clusterType == MiniClusterType.druidLocal
+ || clusterType == MiniClusterType.druid) {
final String tempDir = System.getProperty("test.tmp.dir");
- druidCluster = new MiniDruidCluster("mini-druid",
- logDir,
- tempDir,
- setup.zkPort,
- Utilities.jarFinderGetJar(MiniDruidCluster.class)
- );
+ druidCluster =
+ new MiniDruidCluster(clusterType == MiniClusterType.druid ? "mini-druid" : "mini-druid-kafka",
+ logDir,
+ tempDir,
+ setup.zkPort,
+ Utilities.jarFinderGetJar(MiniDruidCluster.class));
final Path druidDeepStorage = fs.makeQualified(new Path(druidCluster.getDeepStorageDir()));
fs.mkdirs(druidDeepStorage);
- conf.set("hive.druid.storage.storageDirectory", druidDeepStorage.toUri().getPath());
- conf.set("hive.druid.metadata.db.type", "derby");
- conf.set("hive.druid.metadata.uri", druidCluster.getMetadataURI());
- final Path scratchDir = fs
- .makeQualified(new Path(System.getProperty("test.tmp.dir"), "druidStagingDir"));
+ final Path scratchDir = fs.makeQualified(new Path(System.getProperty("test.tmp.dir"), "druidStagingDir"));
fs.mkdirs(scratchDir);
conf.set("hive.druid.working.directory", scratchDir.toUri().getPath());
druidCluster.init(conf);
druidCluster.start();
}
- if (clusterType == MiniClusterType.kafka
- || clusterType == MiniClusterType.druidKafka
- || clusterType == MiniClusterType.druidLocal) {
- kafkaCluster = new SingleNodeKafkaCluster("kafka",
- logDir + "/kafka-cluster",
- setup.zkPort
- );
+ if (clusterType == MiniClusterType.kafka || clusterType == MiniClusterType.druidKafka) {
+ kafkaCluster =
+ new SingleNodeKafkaCluster("kafka",
+ System.getProperty("test.tmp.dir") + "/kafka-cluster",
+ setup.zkPort,
+ clusterType == MiniClusterType.kafka ? 9093 : 9092);
kafkaCluster.init(conf);
kafkaCluster.start();
- kafkaCluster.createTopicWithData(
- "test-topic",
- new File(getScriptsDir(), "kafka_init_data.json")
- );
- kafkaCluster.createTopicWithData(
- "wiki_kafka_csv",
- new File(getScriptsDir(), "kafka_init_data.csv")
- );
+ kafkaCluster.createTopicWithData("test-topic", new File(getScriptsDir(), "kafka_init_data.json"));
+ kafkaCluster.createTopicWithData("wiki_kafka_csv", new File(getScriptsDir(), "kafka_init_data.csv"));
kafkaCluster.createTopicWithData("wiki_kafka_avro_table", getAvroRows());
}
if (clusterType.getCoreClusterType() == CoreClusterType.TEZ) {
if (confDir != null && !confDir.isEmpty()) {
- conf.addResource(new URL("file://" + new File(confDir).toURI().getPath()
- + "/tez-site.xml"));
+ conf.addResource(new URL("file://" + new File(confDir).toURI().getPath() + "/tez-site.xml"));
}
int numTrackers = 2;
- if (EnumSet.of(
- MiniClusterType.llap,
+ if (EnumSet.of(MiniClusterType.llap,
MiniClusterType.llap_local,
MiniClusterType.druidLocal,
- MiniClusterType.druidKafka
- ).contains(clusterType)) {
+ MiniClusterType.druidKafka,
+ MiniClusterType.druid,
+ MiniClusterType.kafka).contains(clusterType)) {
llapCluster = LlapItUtils.startAndGetMiniLlapCluster(conf, setup.zooKeeperCluster, confDir);
}
if (EnumSet.of(MiniClusterType.llap_local, MiniClusterType.tez_local, MiniClusterType.druidLocal)
- .contains(clusterType)) {
- mr = shims.getLocalMiniTezCluster(conf,
- clusterType == MiniClusterType.llap_local
- || clusterType == MiniClusterType.druidLocal
- );
+ .contains(clusterType)) {
+ mr =
+ shims.getLocalMiniTezCluster(conf,
+ clusterType == MiniClusterType.llap_local || clusterType == MiniClusterType.druidLocal);
} else {
- mr = shims.getMiniTezCluster(
- conf,
- numTrackers,
- uriString,
- EnumSet.of(MiniClusterType.llap, MiniClusterType.llap_local, MiniClusterType.druidKafka).contains(clusterType)
- );
+ mr =
+ shims.getMiniTezCluster(conf,
+ numTrackers,
+ uriString,
+ EnumSet.of(MiniClusterType.llap,
+ MiniClusterType.llap_local,
+ MiniClusterType.druidKafka,
+ MiniClusterType.druid,
+ MiniClusterType.kafka).contains(clusterType));
}
} else if (clusterType == MiniClusterType.miniSparkOnYarn) {
mr = shims.getMiniSparkCluster(conf, 2, uriString, 1);
@@ -685,42 +667,38 @@ public class QTestUtil {
int numRows = 10;
List<byte[]> events;
final DatumWriter<GenericRecord> writer = new SpecificDatumWriter<>(Wikipedia.getClassSchema());
- events =
- IntStream.rangeClosed(0, numRows)
- .mapToObj(i -> Wikipedia.newBuilder()
- // 1534736225090 -> 08/19/2018 20:37:05
- .setTimestamp(formatter.format(new Timestamp(1534736225090L + 1000 * 3600 * i)))
- .setAdded(i * 300)
- .setDeleted(-i)
- .setIsrobot(i % 2 == 0)
- .setChannel("chanel number " + i)
- .setComment("comment number " + i)
- .setCommentlength(i)
- .setDiffurl(String.format("url %s", i))
- .setFlags("flag")
- .setIsminor(i % 2 > 0)
- .setIsanonymous(i % 3 != 0)
- .setNamespace("namespace")
- .setIsunpatrolled(new Boolean(i % 3 == 0))
- .setIsnew(new Boolean(i % 2 > 0))
- .setPage(String.format("page is %s", i * 100))
- .setDelta(i)
- .setDeltabucket(i * 100.4)
- .setUser("test-user-" + i)
- .build())
- .map(genericRecord -> {
- java.io.ByteArrayOutputStream out = new java.io.ByteArrayOutputStream();
- BinaryEncoder encoder = EncoderFactory.get().binaryEncoder(out, null);
- try {
- writer.write(genericRecord, encoder);
- encoder.flush();
- out.close();
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- return out.toByteArray();
- })
- .collect(Collectors.toList());
+ events = IntStream.rangeClosed(0, numRows).mapToObj(i -> Wikipedia.newBuilder()
+ // 1534736225090 -> 08/19/2018 20:37:05
+ .setTimestamp(formatter.format(new Timestamp(1534736225090L + 1000 * 3600 * i)))
+ .setAdded(i * 300)
+ .setDeleted(-i)
+ .setIsrobot(i % 2 == 0)
+ .setChannel("chanel number " + i)
+ .setComment("comment number " + i)
+ .setCommentlength(i)
+ .setDiffurl(String.format("url %s", i))
+ .setFlags("flag")
+ .setIsminor(i % 2 > 0)
+ .setIsanonymous(i % 3 != 0)
+ .setNamespace("namespace")
+ .setIsunpatrolled(new Boolean(i % 3 == 0))
+ .setIsnew(new Boolean(i % 2 > 0))
+ .setPage(String.format("page is %s", i * 100))
+ .setDelta(i)
+ .setDeltabucket(i * 100.4)
+ .setUser("test-user-" + i)
+ .build()).map(genericRecord -> {
+ java.io.ByteArrayOutputStream out = new java.io.ByteArrayOutputStream();
+ BinaryEncoder encoder = EncoderFactory.get().binaryEncoder(out, null);
+ try {
+ writer.write(genericRecord, encoder);
+ encoder.flush();
+ out.close();
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ return out.toByteArray();
+ }).collect(Collectors.toList());
return events;
}
@@ -764,8 +742,9 @@ public class QTestUtil {
}
public String readEntireFileIntoString(File queryFile) throws IOException {
- InputStreamReader isr = new InputStreamReader(
- new BufferedInputStream(new FileInputStream(queryFile)), QTestUtil.UTF_8);
+ InputStreamReader
+ isr =
+ new InputStreamReader(new BufferedInputStream(new FileInputStream(queryFile)), QTestUtil.UTF_8);
StringWriter sw = new StringWriter();
try {
IOUtils.copy(isr, sw);
@@ -781,7 +760,7 @@ public class QTestUtil {
addFile(new File(queryFile), false);
}
- public void addFile(File qf, boolean partial) throws IOException {
+ public void addFile(File qf, boolean partial) throws IOException {
String query = readEntireFileIntoString(qf);
qMap.put(qf.getName(), query);
if (partial) {
@@ -859,8 +838,7 @@ public class QTestUtil {
return;
}
- conf.set("hive.metastore.filter.hook",
- "org.apache.hadoop.hive.metastore.DefaultMetaStoreFilterHookImpl");
+ conf.set("hive.metastore.filter.hook", "org.apache.hadoop.hive.metastore.DefaultMetaStoreFilterHookImpl");
db = Hive.get(conf);
// First delete any MVs to avoid race conditions
@@ -921,10 +899,10 @@ public class QTestUtil {
SessionState.get().setCurrentDatabase(DEFAULT_DATABASE_NAME);
List<String> roleNames = db.getAllRoleNames();
- for (String roleName : roleNames) {
- if (!"PUBLIC".equalsIgnoreCase(roleName) && !"ADMIN".equalsIgnoreCase(roleName)) {
- db.dropRole(roleName);
- }
+ for (String roleName : roleNames) {
+ if (!"PUBLIC".equalsIgnoreCase(roleName) && !"ADMIN".equalsIgnoreCase(roleName)) {
+ db.dropRole(roleName);
+ }
}
}
@@ -942,7 +920,8 @@ public class QTestUtil {
// renew the metastore since the cluster type is unencrypted
db = Hive.get(conf); // propagate new conf to meta store
- HiveConf.setVar(conf, HiveConf.ConfVars.HIVE_AUTHENTICATOR_MANAGER,
+ HiveConf.setVar(conf,
+ HiveConf.ConfVars.HIVE_AUTHENTICATOR_MANAGER,
"org.apache.hadoop.hive.ql.security.DummyAuthenticator");
CliSessionState ss = new CliSessionState(conf);
ss.in = System.in;
@@ -959,6 +938,7 @@ public class QTestUtil {
setSessionOutputs("that_shouldnt_happen_there", ss, outf);
}
+
/**
* Clear out any side effects of running tests
*/
@@ -989,7 +969,7 @@ public class QTestUtil {
public void cleanUp(String fileName) throws Exception {
boolean canReuseSession = (fileName == null) || !qNoSessionReuseQuerySet.contains(fileName);
- if(!isSessionStateStarted) {
+ if (!isSessionStateStarted) {
startSessionState(canReuseSession);
}
if (System.getenv(QTEST_LEAVE_FILES) != null) {
@@ -1008,8 +988,8 @@ public class QTestUtil {
FileSystem fs = p.getFileSystem(conf);
try {
- FileStatus [] ls = fs.listStatus(p);
- for (int i=0; (ls != null) && (i<ls.length); i++) {
+ FileStatus[] ls = fs.listStatus(p);
+ for (int i = 0; (ls != null) && (i < ls.length); i++) {
fs.delete(ls[i].getPath(), true);
}
} catch (FileNotFoundException e) {
@@ -1044,7 +1024,7 @@ public class QTestUtil {
public void createSources(String fileName) throws Exception {
boolean canReuseSession = (fileName == null) || !qNoSessionReuseQuerySet.contains(fileName);
- if(!isSessionStateStarted) {
+ if (!isSessionStateStarted) {
startSessionState(canReuseSession);
}
@@ -1174,8 +1154,7 @@ public class QTestUtil {
return outf.getAbsolutePath();
}
- private void setSessionOutputs(String fileName, CliSessionState ss, File outf)
- throws Exception {
+ private void setSessionOutputs(String fileName, CliSessionState ss, File outf) throws Exception {
OutputStream fo = new BufferedOutputStream(new FileOutputStream(outf));
if (ss.out != null) {
ss.out.flush();
@@ -1196,10 +1175,8 @@ public class QTestUtil {
ss.setIsSilent(true);
}
- private void restartSessions(boolean canReuseSession, CliSessionState ss, SessionState oldSs)
- throws IOException {
- if (oldSs != null && canReuseSession
- && clusterType.getCoreClusterType() == CoreClusterType.TEZ) {
+ private void restartSessions(boolean canReuseSession, CliSessionState ss, SessionState oldSs) throws IOException {
+ if (oldSs != null && canReuseSession && clusterType.getCoreClusterType() == CoreClusterType.TEZ) {
// Copy the tezSessionState from the old CliSessionState.
TezSessionState tezSessionState = oldSs.getTezSession();
oldSs.setTezSession(null);
@@ -1217,7 +1194,8 @@ public class QTestUtil {
private CliSessionState startSessionState(boolean canReuseSession) throws IOException {
- HiveConf.setVar(conf, HiveConf.ConfVars.HIVE_AUTHENTICATOR_MANAGER,
+ HiveConf.setVar(conf,
+ HiveConf.ConfVars.HIVE_AUTHENTICATOR_MANAGER,
"org.apache.hadoop.hive.ql.security.DummyAuthenticator");
String execEngine = conf.get("hive.execution.engine");
@@ -1340,10 +1318,10 @@ public class QTestUtil {
//replace ${hiveconf:hive.metastore.warehouse.dir} with actual dir if existed.
//we only want the absolute path, so remove the header, such as hdfs://localhost:57145
- String wareHouseDir = SessionState.get().getConf().getVar(ConfVars.METASTOREWAREHOUSE)
- .replaceAll("^[a-zA-Z]+://.*?:\\d+", "");
- commandArgs = commandArgs.replaceAll("\\$\\{hiveconf:hive\\.metastore\\.warehouse\\.dir\\}",
- wareHouseDir);
+ String
+ wareHouseDir =
+ SessionState.get().getConf().getVar(ConfVars.METASTOREWAREHOUSE).replaceAll("^[a-zA-Z]+://.*?:\\d+", "");
+ commandArgs = commandArgs.replaceAll("\\$\\{hiveconf:hive\\.metastore\\.warehouse\\.dir\\}", wareHouseDir);
if (SessionState.get() != null) {
SessionState.get().setLastCommand(commandName + " " + commandArgs.trim());
@@ -1358,8 +1336,9 @@ public class QTestUtil {
int rc = response.getResponseCode();
if (rc != 0) {
- SessionState.getConsole().printError(response.toString(), response.getException() != null ?
- Throwables.getStackTraceAsString(response.getException()) : "");
+ SessionState.getConsole()
+ .printError(response.toString(),
+ response.getException() != null ? Throwables.getStackTraceAsString(response.getException()) : "");
}
return rc;
@@ -1372,20 +1351,20 @@ public class QTestUtil {
}
private CommandProcessor getTestCommand(final String commandName) throws SQLException {
- HiveCommand testCommand = HiveCommand.find(new String[]{commandName}, HiveCommand.ONLY_FOR_TESTING);
+ HiveCommand testCommand = HiveCommand.find(new String[]{ commandName }, HiveCommand.ONLY_FOR_TESTING);
if (testCommand == null) {
return null;
}
- return CommandProcessorFactory
- .getForHiveCommandInternal(new String[]{commandName}, SessionState.get().getConf(),
+ return CommandProcessorFactory.getForHiveCommandInternal(new String[]{ commandName },
+ SessionState.get().getConf(),
testCommand.isOnlyForTesting());
}
- private void enableTestOnlyCmd(HiveConf conf){
+ private void enableTestOnlyCmd(HiveConf conf) {
StringBuilder securityCMDs = new StringBuilder(conf.getVar(HiveConf.ConfVars.HIVE_SECURITY_COMMAND_WHITELIST));
- for(String c : testOnlyCommands){
+ for (String c : testOnlyCommands) {
securityCMDs.append(",");
securityCMDs.append(c);
}
@@ -1394,7 +1373,7 @@ public class QTestUtil {
private boolean isCommandUsedForTesting(final String command) {
String commandName = command.trim().split("\\s+")[0];
- HiveCommand testCommand = HiveCommand.find(new String[]{commandName}, HiveCommand.ONLY_FOR_TESTING);
+ HiveCommand testCommand = HiveCommand.find(new String[]{ commandName }, HiveCommand.ONLY_FOR_TESTING);
return testCommand != null;
}
@@ -1441,8 +1420,7 @@ public class QTestUtil {
outfd.write(e.getMessage());
outfd.close();
- QTestProcessExecResult result = executeDiffCommand(outf.getPath(), expf, false,
- qSortSet.contains(qf.getName()));
+ QTestProcessExecResult result = executeDiffCommand(outf.getPath(), expf, false, qSortSet.contains(qf.getName()));
if (overWrite) {
overwriteResults(outf.getPath(), expf);
return QTestProcessExecResult.createWithoutOutput(0);
@@ -1464,8 +1442,13 @@ public class QTestUtil {
FileWriter outfd = new FileWriter(outf, true);
- outfd
- .write("FAILED: " + e.getClass().getSimpleName() + " " + e.getClass().getName() + ": " + e.getMessage() + "\n");
+ outfd.write("FAILED: "
+ + e.getClass().getSimpleName()
+ + " "
+ + e.getClass().getName()
+ + ": "
+ + e.getMessage()
+ + "\n");
outfd.close();
QTestProcessExecResult result = executeDiffCommand(outf.getPath(), expf, false, qSortSet.contains(qf.getName()));
@@ -1480,7 +1463,8 @@ public class QTestUtil {
/**
* Given the current configurations (e.g., hadoop version and execution mode), return
* the correct file name to compare with the current test run output.
- * @param outDir The directory where the reference log files are stored.
+ *
+ * @param outDir The directory where the reference log files are stored.
* @param testName The test file name (terminated by ".out").
* @return The file name appended with the configuration values if it exists.
*/
@@ -1496,7 +1480,7 @@ public class QTestUtil {
stack.push(sb.toString());
// example file names are input1.q.out_mr_0.17 or input2.q.out_0.17
- for (String s: configs) {
+ for (String s : configs) {
sb.append('_');
sb.append(s);
stack.push(sb.toString());
@@ -1509,11 +1493,11 @@ public class QTestUtil {
break;
}
}
- return ret;
+ return ret;
}
public QTestProcessExecResult checkCliDriverResults(String tname) throws Exception {
- assert(qMap.containsKey(tname));
+ assert (qMap.containsKey(tname));
String outFileExtension = getOutFileExtension(tname);
String outFileName = outPath(outDir, tname + outFileExtension);
@@ -1521,9 +1505,7 @@ public class QTestUtil {
File f = new File(logDir, tname + outFileExtension);
qOutProcessor.maskPatterns(f.getPath(), tname);
- QTestProcessExecResult exitVal = executeDiffCommand(f.getPath(),
- outFileName, false,
- qSortSet.contains(tname));
+ QTestProcessExecResult exitVal = executeDiffCommand(f.getPath(), outFileName, false, qSortSet.contains(tname));
if (overWrite) {
overwriteResults(f.getPath(), outFileName);
@@ -1533,14 +1515,14 @@ public class QTestUtil {
return exitVal;
}
- public QTestProcessExecResult checkCompareCliDriverResults(String tname, List<String> outputs)
- throws Exception {
+ public QTestProcessExecResult checkCompareCliDriverResults(String tname, List<String> outputs) throws Exception {
assert outputs.size() > 1;
qOutProcessor.maskPatterns(outputs.get(0), tname);
for (int i = 1; i < outputs.size(); ++i) {
qOutProcessor.maskPatterns(outputs.get(i), tname);
- QTestProcessExecResult result = executeDiffCommand(
- outputs.get(i - 1), outputs.get(i), false, qSortSet.contains(tname));
+ QTestProcessExecResult
+ result =
+ executeDiffCommand(outputs.get(i - 1), outputs.get(i), false, qSortSet.contains(tname));
if (result.getReturnCode() != 0) {
System.out.println("Files don't match: " + outputs.get(i - 1) + " and " + outputs.get(i));
return result;
@@ -1554,21 +1536,16 @@ public class QTestUtil {
// once Hive uses JAVA 7.
System.out.println("Overwriting results " + inFileName + " to " + outFileName);
int result = executeCmd(new String[]{
- "cp",
- getQuotedString(inFileName),
- getQuotedString(outFileName)
- }).getReturnCode();
+ "cp", getQuotedString(inFileName), getQuotedString(outFileName) }).getReturnCode();
if (result != 0) {
- throw new IllegalStateException("Unexpected error while overwriting " +
- inFileName + " with " + outFileName);
+ throw new IllegalStateException("Unexpected error while overwriting " + inFileName + " with " + outFileName);
}
}
private static QTestProcessExecResult executeDiffCommand(String inFileName,
String outFileName,
boolean ignoreWhiteSpace,
- boolean sortResults
- ) throws Exception {
+ boolean sortResults) throws Exception {
QTestProcessExecResult result;
@@ -1617,9 +1594,7 @@ public class QTestUtil {
private static void sortFiles(String in, String out) throws Exception {
int result = executeCmd(new String[]{
- "sort",
- getQuotedString(in),
- }, out, null).getReturnCode();
+ "sort", getQuotedString(in), }, out, null).getReturnCode();
if (result != 0) {
throw new IllegalStateException("Unexpected error while sorting " + in);
}
@@ -1633,22 +1608,25 @@ public class QTestUtil {
return executeCmd(args, null, null);
}
- private static QTestProcessExecResult executeCmd(Collection<String> args, String outFile,
- String errFile) throws Exception {
+ private static QTestProcessExecResult executeCmd(Collection<String> args, String outFile, String errFile)
+ throws Exception {
String[] cmdArray = args.toArray(new String[args.size()]);
return executeCmd(cmdArray, outFile, errFile);
}
- private static QTestProcessExecResult executeCmd(String[] args, String outFile,
- String errFile) throws Exception {
+ private static QTestProcessExecResult executeCmd(String[] args, String outFile, String errFile) throws Exception {
System.out.println("Running: " + org.apache.commons.lang.StringUtils.join(args, ' '));
- PrintStream out = outFile == null ?
- SessionState.getConsole().getChildOutStream() :
- new PrintStream(new FileOutputStream(outFile), true, "UTF-8");
- PrintStream err = errFile == null ?
- SessionState.getConsole().getChildErrStream() :
- new PrintStream(new FileOutputStream(errFile), true, "UTF-8");
+ PrintStream
+ out =
+ outFile == null ?
+ SessionState.getConsole().getChildOutStream() :
+ new PrintStream(new FileOutputStream(outFile), true, "UTF-8");
+ PrintStream
+ err =
+ errFile == null ?
+ SessionState.getConsole().getChildErrStream() :
+ new PrintStream(new FileOutputStream(errFile), true, "UTF-8");
Process executor = Runtime.getRuntime().exec(args);
@@ -1678,7 +1656,7 @@ public class QTestUtil {
create(result, new String(bos.toByteArray(), StandardCharsets.UTF_8));
}
- private static String getQuotedString(String str){
+ private static String getQuotedString(String str) {
return str;
}
@@ -1704,8 +1682,7 @@ public class QTestUtil {
* QTestSetup defines test fixtures which are reused across testcases,
* and are needed before any test can be run
*/
- public static class QTestSetup
- {
+ public static class QTestSetup {
private MiniZooKeeperCluster zooKeeperCluster = null;
private int zkPort;
private ZooKeeper zooKeeper;
@@ -1717,7 +1694,7 @@ public class QTestUtil {
if (zooKeeperCluster == null) {
//create temp dir
- String tmpBaseDir = System.getProperty(TEST_TMP_DIR_PROPERTY);
+ String tmpBaseDir = System.getProperty(TEST_TMP_DIR_PROPERTY);
File tmpDir = Files.createTempDirectory(Paths.get(tmpBaseDir), "tmp_").toFile();
zooKeeperCluster = new MiniZooKeeperCluster();
@@ -1728,10 +1705,11 @@ public class QTestUtil {
zooKeeper.close();
}
- int sessionTimeout = (int) conf.getTimeVar(HiveConf.ConfVars.HIVE_ZOOKEEPER_SESSION_TIMEOUT, TimeUnit.MILLISECONDS);
+ int
+ sessionTimeout =
+ (int) conf.getTimeVar(HiveConf.ConfVars.HIVE_ZOOKEEPER_SESSION_TIMEOUT, TimeUnit.MILLISECONDS);
zooKeeper = new ZooKeeper("localhost:" + zkPort, sessionTimeout, new Watcher() {
- @Override
- public void process(WatchedEvent arg0) {
+ @Override public void process(WatchedEvent arg0) {
}
});
@@ -1764,7 +1742,6 @@ public class QTestUtil {
/**
* QTRunner: Runnable class for running a single query file.
- *
**/
public static class QTRunner implements Runnable {
private final QTestUtil qt;
@@ -1775,8 +1752,7 @@ public class QTestUtil {
this.file = file;
}
- @Override
- public void run() {
+ @Override public void run() {
try {
qt.startSessionState(false);
// assumption is that environment has already been cleaned once globally
@@ -1784,8 +1760,7 @@ public class QTestUtil {
qt.cliInit(file);
qt.executeClient(file.getName());
} catch (Throwable e) {
- System.err.println("Query file " + file.getName() + " failed with exception "
- + e.getMessage());
+ System.err.println("Query file " + file.getName() + " failed with exception " + e.getMessage());
e.printStackTrace();
outputTestFailureHelpMessage();
}
@@ -1795,31 +1770,30 @@ public class QTestUtil {
/**
* Setup to execute a set of query files. Uses QTestUtil to do so.
*
- * @param qfiles
- * array of input query files containing arbitrary number of hive
- * queries
- * @param resDir
- * output directory
- * @param logDir
- * log directory
+ * @param qfiles array of input query files containing arbitrary number of hive
+ * queries
+ * @param resDir output directory
+ * @param logDir log directory
* @return one QTestUtil for each query file
*/
- public static QTestUtil[] queryListRunnerSetup(File[] qfiles, String resDir,
- String logDir, String initScript, String cleanupScript) throws Exception
- {
+ public static QTestUtil[] queryListRunnerSetup(File[] qfiles,
+ String resDir,
+ String logDir,
+ String initScript,
+ String cleanupScript) throws Exception {
QTestUtil[] qt = new QTestUtil[qfiles.length];
for (int i = 0; i < qfiles.length; i++) {
- qt[i] = new QTestUtil(
- QTestArguments.QTestArgumentsBuilder.instance()
- .withOutDir(resDir)
- .withLogDir(logDir)
- .withClusterType(MiniClusterType.none)
- .withConfDir(null)
- .withInitScript(initScript == null ? defaultInitScript : initScript)
- .withCleanupScript(cleanupScript == null ? defaultCleanupScript : cleanupScript)
- .withLlapIo(false)
- .build());
+ qt[i] =
+ new QTestUtil(QTestArguments.QTestArgumentsBuilder.instance()
+ .withOutDir(resDir)
+ .withLogDir(logDir)
+ .withClusterType(MiniClusterType.none)
+ .withConfDir(null)
+ .withInitScript(initScript == null ? defaultInitScript : initScript)
+ .withCleanupScript(cleanupScript == null ? defaultCleanupScript : cleanupScript)
+ .withLlapIo(false)
+ .build());
qt[i].addFile(qfiles[i], false);
qt[i].clearTestSideEffects();
@@ -1831,16 +1805,12 @@ public class QTestUtil {
/**
* Executes a set of query files in sequence.
*
- * @param qfiles
- * array of input query files containing arbitrary number of hive
- * queries
- * @param qt
- * array of QTestUtils, one per qfile
+ * @param qfiles array of input query files containing arbitrary number of hive
+ * queries
+ * @param qt array of QTestUtils, one per qfile
* @return true if all queries passed, false otw
*/
- public static boolean queryListRunnerSingleThreaded(File[] qfiles, QTestUtil[] qt)
- throws Exception
- {
+ public static boolean queryListRunnerSingleThreaded(File[] qfiles, QTestUtil[] qt) throws Exception {
boolean failed = false;
qt[0].cleanUp();
qt[0].createSources();
@@ -1869,21 +1839,16 @@ public class QTestUtil {
/**
* Executes a set of query files parallel.
- *
+ * <p>
* Each query file is run in a separate thread. The caller has to arrange
* that different query files do not collide (in terms of destination tables)
*
- * @param qfiles
- * array of input query files containing arbitrary number of hive
- * queries
- * @param qt
- * array of QTestUtils, one per qfile
+ * @param qfiles array of input query files containing arbitrary number of hive
+ * queries
+ * @param qt array of QTestUtils, one per qfile
* @return true if all queries passed, false otw
- *
*/
- public static boolean queryListRunnerMultiThreaded(File[] qfiles, QTestUtil[] qt)
- throws Exception
- {
+ public static boolean queryListRunnerMultiThreaded(File[] qfiles, QTestUtil[] qt) throws Exception {
boolean failed = false;
// in multithreaded mode - do cleanup/initialization just once
@@ -1925,10 +1890,9 @@ public class QTestUtil {
}
public static void outputTestFailureHelpMessage() {
- System.err.println(
- "See ./ql/target/tmp/log/hive.log or ./itests/qtest/target/tmp/log/hive.log, or check " +
- "./ql/target/surefire-reports or ./itests/qtest/target/surefire-reports/ for specific " +
- "test cases logs.");
+ System.err.println("See ./ql/target/tmp/log/hive.log or ./itests/qtest/target/tmp/log/hive.log, or check "
+ + "./ql/target/surefire-reports or ./itests/qtest/target/surefire-reports/ for specific "
+ + "test cases logs.");
System.err.flush();
}
@@ -1952,8 +1916,7 @@ public class QTestUtil {
// Not thread-safe.
System.out.println("Getting versions from " + queryDir);
cachedQvFileList = (new File(queryDir)).list(new FilenameFilter() {
- @Override
- public boolean accept(File dir, String name) {
+ @Override public boolean accept(File dir, String name) {
return name.toLowerCase().endsWith(".qv");
}
});
@@ -1962,8 +1925,7 @@ public class QTestUtil {
}
Arrays.sort(cachedQvFileList, String.CASE_INSENSITIVE_ORDER);
List<String> defaults = getVersionFilesInternal("default");
- cachedDefaultQvFileList = (defaults != null)
- ? ImmutableList.copyOf(defaults) : ImmutableList.<String>of();
+ cachedDefaultQvFileList = (defaults != null) ? ImmutableList.copyOf(defaults) : ImmutableList.<String>of();
}
private static List<String> getVersionFilesInternal(String tname) {
@@ -1992,25 +1954,34 @@ public class QTestUtil {
public void failed(int ecode, String fname, String debugHint) {
String command = SessionState.get() != null ? SessionState.get().getLastCommand() : null;
- String message = "Client execution failed with error code = " + ecode +
- (command != null ? " running \"" + command : "") + "\" fname=" + fname + " " +
- (debugHint != null ? debugHint : "");
+ String
+ message =
+ "Client execution failed with error code = "
+ + ecode
+ + (command != null ? " running \"" + command : "")
+ + "\" fname="
+ + fname
+ + " "
+ + (debugHint != null ? debugHint : "");
LOG.error(message);
Assert.fail(message);
}
// for negative tests, which is succeeded.. no need to print the query string
public void failed(String fname, String debugHint) {
- Assert.fail(
- "Client Execution was expected to fail, but succeeded with error code 0 for fname=" +
- fname + (debugHint != null ? (" " + debugHint) : ""));
+ Assert.fail("Client Execution was expected to fail, but succeeded with error code 0 for fname=" + fname + (debugHint
+ != null ? (" " + debugHint) : ""));
}
public void failedDiff(int ecode, String fname, String debugHint) {
- String message =
- "Client Execution succeeded but contained differences " +
- "(error code = " + ecode + ") after executing " +
- fname + (debugHint != null ? (" " + debugHint) : "");
+ String
+ message =
+ "Client Execution succeeded but contained differences "
+ + "(error code = "
+ + ecode
+ + ") after executing "
+ + fname
+ + (debugHint != null ? (" " + debugHint) : "");
LOG.error(message);
Assert.fail(message);
}
@@ -2019,10 +1990,8 @@ public class QTestUtil {
String command = SessionState.get() != null ? SessionState.get().getLastCommand() : null;
System.err.println("Failed query: " + fname);
System.err.flush();
- Assert.fail("Unexpected exception " +
- org.apache.hadoop.util.StringUtils.stringifyException(e) + "\n" +
- (command != null ? " running " + command : "") +
- (debugHint != null ? debugHint : ""));
+ Assert.fail("Unexpected exception " + org.apache.hadoop.util.StringUtils.stringifyException(e) + "\n" + (command
+ != null ? " running " + command : "") + (debugHint != null ? debugHint : ""));
}
public QOutProcessor getQOutProcessor() {
@@ -2035,6 +2004,7 @@ public class QTestUtil {
/**
* Should deleted test tables have their data purged.
+ *
* @return true if data should be purged
*/
private static boolean fsNeedsPurge(FsType type) {
http://git-wip-us.apache.org/repos/asf/hive/blob/8e30b5e0/ql/src/test/queries/clientpositive/druidmini_test_insert.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/druidmini_test_insert.q b/ql/src/test/queries/clientpositive/druidmini_test_insert.q
index 28fa6d7..96644f7 100644
--- a/ql/src/test/queries/clientpositive/druidmini_test_insert.q
+++ b/ql/src/test/queries/clientpositive/druidmini_test_insert.q
@@ -153,4 +153,4 @@ select * from druid_test_table_n9 where `__time` = cast('2015-03-12 00:00:00' as
DROP TABLE test_base_table;
DROP TABLE druid_test_table_n9;
-drop database druid_test_dst;
+drop database druid_test_dst;
http://git-wip-us.apache.org/repos/asf/hive/blob/8e30b5e0/ql/src/test/queries/clientpositive/kafka_storage_handler.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/kafka_storage_handler.q b/ql/src/test/queries/clientpositive/kafka_storage_handler.q
index bab13cb..c1d5f9d 100644
--- a/ql/src/test/queries/clientpositive/kafka_storage_handler.q
+++ b/ql/src/test/queries/clientpositive/kafka_storage_handler.q
@@ -8,7 +8,7 @@ STORED BY 'org.apache.hadoop.hive.kafka.KafkaStorageHandler'
WITH SERDEPROPERTIES ("timestamp.formats"="yyyy-MM-dd\'T\'HH:mm:ss\'Z\'")
TBLPROPERTIES
("kafka.topic" = "test-topic",
-"kafka.bootstrap.servers"="localhost:9092",
+"kafka.bootstrap.servers"="localhost:9093",
"kafka.serde.class"="org.apache.hadoop.hive.serde2.JsonSerDe")
;
@@ -142,7 +142,7 @@ CREATE EXTERNAL TABLE kafka_table_2
STORED BY 'org.apache.hadoop.hive.kafka.KafkaStorageHandler'
TBLPROPERTIES
("kafka.topic" = "test-topic",
-"kafka.bootstrap.servers"="localhost:9092");
+"kafka.bootstrap.servers"="localhost:9093");
Select `__partition`, `__offset`, `__time`, `page`, `user`, `language`, `country`,`continent`, `namespace`, `newPage` ,
`unpatrolled` , `anonymous` , `robot` , added , deleted , delta
@@ -154,7 +154,7 @@ CREATE EXTERNAL TABLE wiki_kafka_avro_table
STORED BY 'org.apache.hadoop.hive.kafka.KafkaStorageHandler'
TBLPROPERTIES
("kafka.topic" = "wiki_kafka_avro_table",
-"kafka.bootstrap.servers"="localhost:9092",
+"kafka.bootstrap.servers"="localhost:9093",
"kafka.serde.class"="org.apache.hadoop.hive.serde2.avro.AvroSerDe",
'avro.schema.literal'='{
"type" : "record",
@@ -242,7 +242,7 @@ STORED BY 'org.apache.hadoop.hive.kafka.KafkaStorageHandler'
WITH SERDEPROPERTIES ("timestamp.formats"="yyyy-MM-dd\'T\'HH:mm:ss\'Z\'")
TBLPROPERTIES
("kafka.topic" = "test-topic-write-json",
-"kafka.bootstrap.servers"="localhost:9092",
+"kafka.bootstrap.servers"="localhost:9093",
"kafka.serde.class"="org.apache.hadoop.hive.serde2.JsonSerDe")
;
@@ -275,7 +275,7 @@ CREATE EXTERNAL TABLE kafka_table_csv
STORED BY 'org.apache.hadoop.hive.kafka.KafkaStorageHandler'
TBLPROPERTIES
("kafka.topic" = "test-topic-write-csv",
-"kafka.bootstrap.servers"="localhost:9092",
+"kafka.bootstrap.servers"="localhost:9093",
"kafka.serde.class"="org.apache.hadoop.hive.serde2.OpenCSVSerde");
ALTER TABLE kafka_table_csv SET TBLPROPERTIES ("hive.kafka.optimistic.commit"="false", "kafka.write.semantic"="EXACTLY_ONCE");
http://git-wip-us.apache.org/repos/asf/hive/blob/8e30b5e0/ql/src/test/results/clientpositive/druid/druidmini_test_insert.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/druid/druidmini_test_insert.q.out b/ql/src/test/results/clientpositive/druid/druidmini_test_insert.q.out
index b74d33c..bb309e5 100644
--- a/ql/src/test/results/clientpositive/druid/druidmini_test_insert.q.out
+++ b/ql/src/test/results/clientpositive/druid/druidmini_test_insert.q.out
@@ -865,11 +865,11 @@ POSTHOOK: query: DROP TABLE druid_test_table_n9
POSTHOOK: type: DROPTABLE
POSTHOOK: Input: druid_test_dst@druid_test_table_n9
POSTHOOK: Output: druid_test_dst@druid_test_table_n9
-PREHOOK: query: drop database druid_test_dst
+PREHOOK: query: drop database druid_test_dst
PREHOOK: type: DROPDATABASE
PREHOOK: Input: database:druid_test_dst
PREHOOK: Output: database:druid_test_dst
-POSTHOOK: query: drop database druid_test_dst
+POSTHOOK: query: drop database druid_test_dst
POSTHOOK: type: DROPDATABASE
POSTHOOK: Input: database:druid_test_dst
POSTHOOK: Output: database:druid_test_dst