You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@phoenix.apache.org by el...@apache.org on 2018/04/19 03:49:58 UTC
[2/3] phoenix git commit: PHOENIX-4423 Hive 2.3.0 support
http://git-wip-us.apache.org/repos/asf/phoenix/blob/83825dec/phoenix-hive/src/it/java/org/apache/hadoop/hive/ql/QTestUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-hive/src/it/java/org/apache/hadoop/hive/ql/QTestUtil.java b/phoenix-hive/src/it/java/org/apache/hadoop/hive/ql/QTestUtil.java
new file mode 100644
index 0000000..2341804
--- /dev/null
+++ b/phoenix-hive/src/it/java/org/apache/hadoop/hive/ql/QTestUtil.java
@@ -0,0 +1,2489 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql;
+
+import static org.apache.hadoop.hive.metastore.Warehouse.DEFAULT_DATABASE_NAME;
+
+import java.io.BufferedInputStream;
+import java.io.BufferedOutputStream;
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.FileReader;
+import java.io.FileWriter;
+import java.io.FilenameFilter;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.OutputStream;
+import java.io.OutputStreamWriter;
+import java.io.PrintStream;
+import java.io.Serializable;
+import java.io.StringWriter;
+import java.net.URL;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.FileSystems;
+import java.nio.file.Files;
+import java.nio.file.StandardOpenOption;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Comparator;
+import java.util.Deque;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Set;
+import java.util.SortedMap;
+import java.util.TreeMap;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.TimeUnit;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.stream.Stream;
+
+import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.io.output.ByteArrayOutputStream;
+import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.tuple.ImmutablePair;
+import org.apache.commons.lang3.tuple.Pair;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.zookeeper.MiniZooKeeperCluster;
+import org.apache.hadoop.hive.cli.CliDriver;
+import org.apache.hadoop.hive.cli.CliSessionState;
+import org.apache.hadoop.hive.common.io.CachingPrintStream;
+import org.apache.hadoop.hive.common.io.DigestPrintStream;
+import org.apache.hadoop.hive.common.io.SortAndDigestPrintStream;
+import org.apache.hadoop.hive.common.io.SortPrintStream;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
+import org.apache.hadoop.hive.llap.io.api.LlapProxy;
+import org.apache.hadoop.hive.metastore.Warehouse;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
+import org.apache.hadoop.hive.ql.cache.results.QueryResultsCache;
+import org.apache.hadoop.hive.ql.exec.FunctionRegistry;
+import org.apache.hadoop.hive.ql.exec.Task;
+import org.apache.hadoop.hive.ql.exec.Utilities;
+import org.apache.hadoop.hive.ql.exec.spark.session.SparkSession;
+import org.apache.hadoop.hive.ql.exec.spark.session.SparkSessionManagerImpl;
+import org.apache.hadoop.hive.ql.exec.tez.TezSessionState;
+import org.apache.hadoop.hive.ql.lockmgr.zookeeper.CuratorFrameworkSingleton;
+import org.apache.hadoop.hive.ql.lockmgr.zookeeper.ZooKeeperHiveLockManager;
+import org.apache.hadoop.hive.ql.metadata.Hive;
+import org.apache.hadoop.hive.ql.metadata.HiveMaterializedViewsRegistry;
+import org.apache.hadoop.hive.ql.metadata.InvalidTableException;
+import org.apache.hadoop.hive.ql.metadata.Table;
+import org.apache.hadoop.hive.ql.parse.ASTNode;
+import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer;
+import org.apache.hadoop.hive.ql.parse.ParseDriver;
+import org.apache.hadoop.hive.ql.parse.ParseException;
+import org.apache.hadoop.hive.ql.parse.SemanticAnalyzer;
+import org.apache.hadoop.hive.ql.parse.SemanticException;
+import org.apache.hadoop.hive.ql.processors.CommandProcessor;
+import org.apache.hadoop.hive.ql.processors.CommandProcessorFactory;
+import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse;
+import org.apache.hadoop.hive.ql.processors.HiveCommand;
+import org.apache.hadoop.hive.ql.session.SessionState;
+import org.apache.hadoop.hive.shims.HadoopShims;
+import org.apache.hadoop.hive.shims.ShimLoader;
+import org.apache.hive.common.util.StreamPrinter;
+import org.apache.logging.log4j.util.Strings;
+import org.apache.tools.ant.BuildException;
+import org.apache.zookeeper.WatchedEvent;
+import org.apache.zookeeper.Watcher;
+import org.apache.zookeeper.ZooKeeper;
+import org.junit.Assert;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.base.Preconditions;
+import com.google.common.base.Throwables;
+import com.google.common.collect.ImmutableList;
+
+import junit.framework.TestSuite;
+
+/**
+ * QTestUtil. Cloned from Hive 3.0.0 as hive doesn't release hive-it-util artifact
+ *
+ */
+public class QTestUtil {
+ public static final String UTF_8 = "UTF-8";
+ public static final String HIVE_ROOT = getHiveRoot();
+ // security property names
+ private static final String SECURITY_KEY_PROVIDER_URI_NAME = "dfs.encryption.key.provider.uri";
+ private static final String CRLF = System.getProperty("line.separator");
+
+ public static final String QTEST_LEAVE_FILES = "QTEST_LEAVE_FILES";
+ private static final Logger LOG = LoggerFactory.getLogger("QTestUtil");
+ private final static String defaultInitScript = "q_test_init.sql";
+ private final static String defaultCleanupScript = "q_test_cleanup.sql";
+ private final String[] testOnlyCommands = new String[]{"crypto"};
+
+ private static final String TEST_TMP_DIR_PROPERTY = "test.tmp.dir"; // typically target/tmp
+ private static final String BUILD_DIR_PROPERTY = "build.dir"; // typically target
+
+ public static final String PATH_HDFS_REGEX = "(hdfs://)([a-zA-Z0-9:/_\\-\\.=])+";
+ public static final String PATH_HDFS_WITH_DATE_USER_GROUP_REGEX = "([a-z]+) ([a-z]+)([ ]+)([0-9]+) ([0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}) " + PATH_HDFS_REGEX;
+
+ private String testWarehouse;
+ private final String testFiles;
+ protected final String outDir;
+ protected String overrideResultsDir;
+ protected final String logDir;
+ private final TreeMap<String, String> qMap;
+ private final Set<String> qSkipSet;
+ private final Set<String> qSortSet;
+ private final Set<String> qSortQuerySet;
+ private final Set<String> qHashQuerySet;
+ private final Set<String> qSortNHashQuerySet;
+ private final Set<String> qNoSessionReuseQuerySet;
+ private final Set<String> qJavaVersionSpecificOutput;
+ private static final String SORT_SUFFIX = ".sorted";
+ private final Set<String> srcTables;
+ private final Set<String> srcUDFs;
+ private final MiniClusterType clusterType;
+ private final FsType fsType;
+ private ParseDriver pd;
+ protected Hive db;
+ protected QueryState queryState;
+ protected HiveConf conf;
+ private IDriver drv;
+ private BaseSemanticAnalyzer sem;
+ protected final boolean overWrite;
+ private CliDriver cliDriver;
+ private HadoopShims.MiniMrShim mr = null;
+ private HadoopShims.MiniDFSShim dfs = null;
+ private FileSystem fs;
+ private HadoopShims.HdfsEncryptionShim hes = null;
+ private String hadoopVer = null;
+ private QTestSetup setup = null;
+ private SparkSession sparkSession = null;
+ private boolean isSessionStateStarted = false;
+ private static final String javaVersion = getJavaVersion();
+
+ private final String initScript;
+ private final String cleanupScript;
+
+
+ public interface SuiteAddTestFunctor {
+ public void addTestToSuite(TestSuite suite, Object setup, String tName);
+ }
+
+ public static Set<String> getSrcTables() {
+ HashSet<String> srcTables = new HashSet<String>();
+ // FIXME: moved default value to here...for now
+ // i think this features is never really used from the command line
+ String defaultTestSrcTables = "src,src1,srcbucket,srcbucket2,src_json,src_thrift," +
+ "src_sequencefile,srcpart,alltypesorc,src_hbase,cbo_t1,cbo_t2,cbo_t3,src_cbo,part," +
+ "lineitem,alltypesparquet";
+ for (String srcTable : System.getProperty("test.src.tables", defaultTestSrcTables).trim().split(",")) {
+ srcTable = srcTable.trim();
+ if (!srcTable.isEmpty()) {
+ srcTables.add(srcTable);
+ }
+ }
+ if (srcTables.isEmpty()) {
+ throw new RuntimeException("Source tables cannot be empty");
+ }
+ return srcTables;
+ }
+
+ /**
+ * Returns the default UDF names which should not be removed when resetting the test database
+ * @return The list of the UDF names not to remove
+ */
+ private Set<String> getSrcUDFs() {
+ HashSet<String> srcUDFs = new HashSet<String>();
+ // FIXME: moved default value to here...for now
+ // i think this features is never really used from the command line
+ String defaultTestSrcUDFs = "qtest_get_java_boolean";
+ for (String srcUDF : System.getProperty("test.src.udfs", defaultTestSrcUDFs).trim().split(","))
+ {
+ srcUDF = srcUDF.trim();
+ if (!srcUDF.isEmpty()) {
+ srcUDFs.add(srcUDF);
+ }
+ }
+ if (srcUDFs.isEmpty()) {
+ throw new RuntimeException("Source UDFs cannot be empty");
+ }
+ return srcUDFs;
+ }
+
+
+
+ public HiveConf getConf() {
+ return conf;
+ }
+
+ public boolean deleteDirectory(File path) {
+ if (path.exists()) {
+ File[] files = path.listFiles();
+ for (File file : files) {
+ if (file.isDirectory()) {
+ deleteDirectory(file);
+ } else {
+ file.delete();
+ }
+ }
+ }
+ return (path.delete());
+ }
+
+ public void copyDirectoryToLocal(Path src, Path dest) throws Exception {
+
+ FileSystem srcFs = src.getFileSystem(conf);
+ FileSystem destFs = dest.getFileSystem(conf);
+ if (srcFs.exists(src)) {
+ FileStatus[] files = srcFs.listStatus(src);
+ for (FileStatus file : files) {
+ String name = file.getPath().getName();
+ Path dfs_path = file.getPath();
+ Path local_path = new Path(dest, name);
+
+ // If this is a source table we do not copy it out
+ if (srcTables.contains(name)) {
+ continue;
+ }
+
+ if (file.isDirectory()) {
+ if (!destFs.exists(local_path)) {
+ destFs.mkdirs(local_path);
+ }
+ copyDirectoryToLocal(dfs_path, local_path);
+ } else {
+ srcFs.copyToLocalFile(dfs_path, local_path);
+ }
+ }
+ }
+ }
+
+ static Pattern mapTok = Pattern.compile("(\\.?)(.*)_map_(.*)");
+ static Pattern reduceTok = Pattern.compile("(.*)(reduce_[^\\.]*)((\\..*)?)");
+
+ public void normalizeNames(File path) throws Exception {
+ if (path.isDirectory()) {
+ File[] files = path.listFiles();
+ for (File file : files) {
+ normalizeNames(file);
+ }
+ } else {
+ Matcher m = reduceTok.matcher(path.getName());
+ if (m.matches()) {
+ String name = m.group(1) + "reduce" + m.group(3);
+ path.renameTo(new File(path.getParent(), name));
+ } else {
+ m = mapTok.matcher(path.getName());
+ if (m.matches()) {
+ String name = m.group(1) + "map_" + m.group(3);
+ path.renameTo(new File(path.getParent(), name));
+ }
+ }
+ }
+ }
+
+ public String getOutputDirectory() {
+ return outDir;
+ }
+
+ public String getLogDirectory() {
+ return logDir;
+ }
+
+ private String getHadoopMainVersion(String input) {
+ if (input == null) {
+ return null;
+ }
+ Pattern p = Pattern.compile("^(\\d+\\.\\d+).*");
+ Matcher m = p.matcher(input);
+ if (m.matches()) {
+ return m.group(1);
+ }
+ return null;
+ }
+
+ public void initConf() throws Exception {
+
+ String vectorizationEnabled = System.getProperty("test.vectorization.enabled");
+ if(vectorizationEnabled != null && vectorizationEnabled.equalsIgnoreCase("true")) {
+ conf.setBoolVar(ConfVars.HIVE_VECTORIZATION_ENABLED, true);
+ }
+
+ // Plug verifying metastore in for testing DirectSQL.
+ conf.setVar(ConfVars.METASTORE_RAW_STORE_IMPL,
+ "org.apache.hadoop.hive.metastore.VerifyingObjectStore");
+
+ if (mr != null) {
+ mr.setupConfiguration(conf);
+
+ // TODO Ideally this should be done independent of whether mr is setup or not.
+ setFsRelatedProperties(conf, fs.getScheme().equals("file"),fs);
+ }
+ conf.set(ConfVars.HIVE_EXECUTION_ENGINE.varname, clusterType.name());
+ }
+
+ private void setFsRelatedProperties(HiveConf conf, boolean isLocalFs, FileSystem fs) {
+ String fsUriString = fs.getUri().toString();
+
+ // Different paths if running locally vs a remote fileSystem. Ideally this difference should not exist.
+ Path warehousePath;
+ Path jarPath;
+ Path userInstallPath;
+ if (isLocalFs) {
+ String buildDir = System.getProperty(BUILD_DIR_PROPERTY);
+ Preconditions.checkState(Strings.isNotBlank(buildDir));
+ Path path = new Path(fsUriString, buildDir);
+
+ // Create a fake fs root for local fs
+ Path localFsRoot = new Path(path, "localfs");
+ warehousePath = new Path(localFsRoot, "warehouse");
+ jarPath = new Path(localFsRoot, "jar");
+ userInstallPath = new Path(localFsRoot, "user_install");
+ } else {
+ // TODO Why is this changed from the default in hive-conf?
+ warehousePath = new Path(fsUriString, "/build/ql/test/data/warehouse/");
+ jarPath = new Path(new Path(fsUriString, "/user"), "hive");
+ userInstallPath = new Path(fsUriString, "/user");
+ }
+
+ warehousePath = fs.makeQualified(warehousePath);
+ jarPath = fs.makeQualified(jarPath);
+ userInstallPath = fs.makeQualified(userInstallPath);
+
+ conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, fsUriString);
+
+ // Remote dirs
+ conf.setVar(ConfVars.METASTOREWAREHOUSE, warehousePath.toString());
+ conf.setVar(ConfVars.HIVE_JAR_DIRECTORY, jarPath.toString());
+ conf.setVar(ConfVars.HIVE_USER_INSTALL_DIR, userInstallPath.toString());
+ // ConfVars.SCRATCHDIR - {test.tmp.dir}/scratchdir
+
+ // Local dirs
+ // ConfVars.LOCALSCRATCHDIR - {test.tmp.dir}/localscratchdir
+
+ // TODO Make sure to cleanup created dirs.
+ }
+
+ private void createRemoteDirs() {
+ assert fs != null;
+ Path warehousePath = fs.makeQualified(new Path(conf.getVar(ConfVars.METASTOREWAREHOUSE)));
+ assert warehousePath != null;
+ Path hiveJarPath = fs.makeQualified(new Path(conf.getVar(ConfVars.HIVE_JAR_DIRECTORY)));
+ assert hiveJarPath != null;
+ Path userInstallPath = fs.makeQualified(new Path(conf.getVar(ConfVars.HIVE_USER_INSTALL_DIR)));
+ assert userInstallPath != null;
+ try {
+ fs.mkdirs(warehousePath);
+ } catch (IOException e) {
+ LOG.error("Failed to create path={}. Continuing. Exception message={}", warehousePath,
+ e.getMessage());
+ }
+ try {
+ fs.mkdirs(hiveJarPath);
+ } catch (IOException e) {
+ LOG.error("Failed to create path={}. Continuing. Exception message={}", warehousePath,
+ e.getMessage());
+ }
+ try {
+ fs.mkdirs(userInstallPath);
+ } catch (IOException e) {
+ LOG.error("Failed to create path={}. Continuing. Exception message={}", warehousePath,
+ e.getMessage());
+ }
+ }
+
+ private enum CoreClusterType {
+ MR,
+ TEZ,
+ SPARK,
+ DRUID
+ }
+
+ public enum FsType {
+ local,
+ hdfs,
+ encrypted_hdfs,
+ }
+
+ public enum MiniClusterType {
+
+ mr(CoreClusterType.MR, FsType.hdfs),
+ tez(CoreClusterType.TEZ, FsType.hdfs),
+ tez_local(CoreClusterType.TEZ, FsType.local),
+ spark(CoreClusterType.SPARK, FsType.local),
+ miniSparkOnYarn(CoreClusterType.SPARK, FsType.hdfs),
+ llap(CoreClusterType.TEZ, FsType.hdfs),
+ llap_local(CoreClusterType.TEZ, FsType.local),
+ none(CoreClusterType.MR, FsType.local),
+ druid(CoreClusterType.DRUID, FsType.hdfs);
+
+
+ private final CoreClusterType coreClusterType;
+ private final FsType defaultFsType;
+
+ MiniClusterType(CoreClusterType coreClusterType, FsType defaultFsType) {
+ this.coreClusterType = coreClusterType;
+ this.defaultFsType = defaultFsType;
+ }
+
+ public CoreClusterType getCoreClusterType() {
+ return coreClusterType;
+ }
+
+ public FsType getDefaultFsType() {
+ return defaultFsType;
+ }
+
+ public static MiniClusterType valueForString(String type) {
+ // Replace this with valueOf.
+ if (type.equals("miniMR")) {
+ return mr;
+ } else if (type.equals("tez")) {
+ return tez;
+ } else if (type.equals("tez_local")) {
+ return tez_local;
+ } else if (type.equals("spark")) {
+ return spark;
+ } else if (type.equals("miniSparkOnYarn")) {
+ return miniSparkOnYarn;
+ } else if (type.equals("llap")) {
+ return llap;
+ } else if (type.equals("llap_local")) {
+ return llap_local;
+ } else if (type.equals("druid")) {
+ return druid;
+ } else {
+ return none;
+ }
+ }
+ }
+
+
+ private String getKeyProviderURI() {
+ // Use the target directory if it is not specified
+ String keyDir = HIVE_ROOT + "ql/target/";
+
+ // put the jks file in the current test path only for test purpose
+ return "jceks://file" + new Path(keyDir, "test.jks").toUri();
+ }
+
+ public QTestUtil(String outDir, String logDir, MiniClusterType clusterType,
+ String confDir, String hadoopVer, String initScript, String cleanupScript,
+ boolean withLlapIo) throws Exception {
+ this(outDir, logDir, clusterType, confDir, hadoopVer, initScript, cleanupScript,
+ withLlapIo, null);
+ }
+
+ public QTestUtil(String outDir, String logDir, MiniClusterType clusterType,
+ String confDir, String hadoopVer, String initScript, String cleanupScript,
+ boolean withLlapIo, FsType fsType)
+ throws Exception {
+ LOG.info("Setting up QTestUtil with outDir="+outDir+", logDir="+logDir+", clusterType="+clusterType+", confDir="+confDir+"," +
+ " hadoopVer="+hadoopVer+", initScript="+initScript+", cleanupScript="+cleanupScript+", withLlapIo="+withLlapIo+"," +
+ " fsType="+fsType+"");
+ Preconditions.checkNotNull(clusterType, "ClusterType cannot be null");
+ if (fsType != null) {
+ this.fsType = fsType;
+ } else {
+ this.fsType = clusterType.getDefaultFsType();
+ }
+ this.outDir = outDir;
+ this.logDir = logDir;
+ this.srcTables=getSrcTables();
+ this.srcUDFs = getSrcUDFs();
+
+ // HIVE-14443 move this fall-back logic to CliConfigs
+ if (confDir != null && !confDir.isEmpty()) {
+ HiveConf.setHiveSiteLocation(new URL("file://"+ new File(confDir).toURI().getPath() + "/hive-site.xml"));
+ MetastoreConf.setHiveSiteLocation(HiveConf.getHiveSiteLocation());
+ System.out.println("Setting hive-site: "+HiveConf.getHiveSiteLocation());
+ }
+
+ queryState = new QueryState.Builder().withHiveConf(new HiveConf(IDriver.class)).build();
+ conf = queryState.getConf();
+ this.hadoopVer = getHadoopMainVersion(hadoopVer);
+ qMap = new TreeMap<String, String>();
+ qSkipSet = new HashSet<String>();
+ qSortSet = new HashSet<String>();
+ qSortQuerySet = new HashSet<String>();
+ qHashQuerySet = new HashSet<String>();
+ qSortNHashQuerySet = new HashSet<String>();
+ qNoSessionReuseQuerySet = new HashSet<String>();
+ qJavaVersionSpecificOutput = new HashSet<String>();
+ this.clusterType = clusterType;
+
+ HadoopShims shims = ShimLoader.getHadoopShims();
+
+ setupFileSystem(shims);
+
+ setup = new QTestSetup();
+ setup.preTest(conf);
+
+ setupMiniCluster(shims, confDir);
+
+ initConf();
+
+ if (withLlapIo && (clusterType == MiniClusterType.none)) {
+ LOG.info("initializing llap IO");
+ LlapProxy.initializeLlapIo(conf);
+ }
+
+
+ // Use the current directory if it is not specified
+ String dataDir = conf.get("test.data.files");
+ if (dataDir == null) {
+ dataDir = new File(".").getAbsolutePath() + "/data/files";
+ }
+ testFiles = dataDir;
+
+ // Use the current directory if it is not specified
+ String scriptsDir = conf.get("test.data.scripts");
+ if (scriptsDir == null) {
+ scriptsDir = new File(".").getAbsolutePath() + "/data/scripts";
+ }
+
+ this.initScript = scriptsDir + File.separator + initScript;
+ this.cleanupScript = scriptsDir + File.separator + cleanupScript;
+
+ overWrite = "true".equalsIgnoreCase(System.getProperty("test.output.overwrite"));
+
+ init();
+ }
+
+ private void setupFileSystem(HadoopShims shims) throws IOException {
+
+ if (fsType == FsType.local) {
+ fs = FileSystem.getLocal(conf);
+ } else if (fsType == FsType.hdfs || fsType == FsType.encrypted_hdfs) {
+ int numDataNodes = 4;
+
+ if (fsType == FsType.encrypted_hdfs) {
+ // Set the security key provider so that the MiniDFS cluster is initialized
+ // with encryption
+ conf.set(SECURITY_KEY_PROVIDER_URI_NAME, getKeyProviderURI());
+ conf.setInt("fs.trash.interval", 50);
+
+ dfs = shims.getMiniDfs(conf, numDataNodes, true, null);
+ fs = dfs.getFileSystem();
+
+ // set up the java key provider for encrypted hdfs cluster
+ hes = shims.createHdfsEncryptionShim(fs, conf);
+
+ LOG.info("key provider is initialized");
+ } else {
+ dfs = shims.getMiniDfs(conf, numDataNodes, true, null);
+ fs = dfs.getFileSystem();
+ }
+ } else {
+ throw new IllegalArgumentException("Unknown or unhandled fsType [" + fsType + "]");
+ }
+ }
+
+ private void setupMiniCluster(HadoopShims shims, String confDir) throws
+ IOException {
+
+ String uriString = fs.getUri().toString();
+
+ if (clusterType.getCoreClusterType() == CoreClusterType.TEZ) {
+ if (confDir != null && !confDir.isEmpty()) {
+ conf.addResource(new URL("file://" + new File(confDir).toURI().getPath()
+ + "/tez-site.xml"));
+ }
+ int numTrackers = 2;
+ if (EnumSet.of(MiniClusterType.llap_local, MiniClusterType.tez_local).contains(clusterType)) {
+ mr = shims.getLocalMiniTezCluster(conf, clusterType == MiniClusterType.llap_local);
+ } else {
+ mr = shims.getMiniTezCluster(conf, numTrackers, uriString,
+ EnumSet.of(MiniClusterType.llap, MiniClusterType.llap_local).contains(clusterType));
+ }
+ } else if (clusterType == MiniClusterType.miniSparkOnYarn) {
+ mr = shims.getMiniSparkCluster(conf, 2, uriString, 1);
+ } else if (clusterType == MiniClusterType.mr) {
+ mr = shims.getMiniMrCluster(conf, 2, uriString, 1);
+ }
+ }
+
+
+ public void shutdown() throws Exception {
+ if (System.getenv(QTEST_LEAVE_FILES) == null) {
+ cleanUp();
+ }
+
+ if (clusterType.getCoreClusterType() == CoreClusterType.TEZ) {
+ SessionState.get().getTezSession().destroy();
+ }
+
+ setup.tearDown();
+ if (sparkSession != null) {
+ try {
+ SparkSessionManagerImpl.getInstance().closeSession(sparkSession);
+ } catch (Exception ex) {
+ LOG.error("Error closing spark session.", ex);
+ } finally {
+ sparkSession = null;
+ }
+ }
+ if (mr != null) {
+ mr.shutdown();
+ mr = null;
+ }
+ FileSystem.closeAll();
+ if (dfs != null) {
+ dfs.shutdown();
+ dfs = null;
+ }
+ Hive.closeCurrent();
+ }
+
+ public String readEntireFileIntoString(File queryFile) throws IOException {
+ InputStreamReader isr = new InputStreamReader(
+ new BufferedInputStream(new FileInputStream(queryFile)), QTestUtil.UTF_8);
+ StringWriter sw = new StringWriter();
+ try {
+ IOUtils.copy(isr, sw);
+ } finally {
+ if (isr != null) {
+ isr.close();
+ }
+ }
+ return sw.toString();
+ }
+
+ public void addFile(String queryFile) throws IOException {
+ addFile(queryFile, false);
+ }
+
+ public void addFile(String queryFile, boolean partial) throws IOException {
+ addFile(new File(queryFile));
+ }
+
+ public void addFile(File qf) throws IOException {
+ addFile(qf, false);
+ }
+
+ public void addFile(File qf, boolean partial) throws IOException {
+ String query = readEntireFileIntoString(qf);
+ qMap.put(qf.getName(), query);
+ if (partial) {
+ return;
+ }
+
+ if(checkHadoopVersionExclude(qf.getName(), query)) {
+ qSkipSet.add(qf.getName());
+ }
+
+ if (checkNeedJavaSpecificOutput(qf.getName(), query)) {
+ qJavaVersionSpecificOutput.add(qf.getName());
+ }
+
+ if (matches(SORT_BEFORE_DIFF, query)) {
+ qSortSet.add(qf.getName());
+ } else if (matches(SORT_QUERY_RESULTS, query)) {
+ qSortQuerySet.add(qf.getName());
+ } else if (matches(HASH_QUERY_RESULTS, query)) {
+ qHashQuerySet.add(qf.getName());
+ } else if (matches(SORT_AND_HASH_QUERY_RESULTS, query)) {
+ qSortNHashQuerySet.add(qf.getName());
+ }
+ if (matches(NO_SESSION_REUSE, query)) {
+ qNoSessionReuseQuerySet.add(qf.getName());
+ }
+ }
+
+ private static final Pattern SORT_BEFORE_DIFF = Pattern.compile("-- SORT_BEFORE_DIFF");
+ private static final Pattern SORT_QUERY_RESULTS = Pattern.compile("-- SORT_QUERY_RESULTS");
+ private static final Pattern HASH_QUERY_RESULTS = Pattern.compile("-- HASH_QUERY_RESULTS");
+ private static final Pattern SORT_AND_HASH_QUERY_RESULTS = Pattern.compile("-- SORT_AND_HASH_QUERY_RESULTS");
+ private static final Pattern NO_SESSION_REUSE = Pattern.compile("-- NO_SESSION_REUSE");
+
+ private boolean matches(Pattern pattern, String query) {
+ Matcher matcher = pattern.matcher(query);
+ if (matcher.find()) {
+ return true;
+ }
+ return false;
+ }
+
+ private boolean checkHadoopVersionExclude(String fileName, String query){
+
+ // Look for a hint to not run a test on some Hadoop versions
+ Pattern pattern = Pattern.compile("-- (EX|IN)CLUDE_HADOOP_MAJOR_VERSIONS\\((.*)\\)");
+
+ boolean excludeQuery = false;
+ boolean includeQuery = false;
+ Set<String> versionSet = new HashSet<String>();
+ String hadoopVer = ShimLoader.getMajorVersion();
+
+ Matcher matcher = pattern.matcher(query);
+
+ // Each qfile may include at most one INCLUDE or EXCLUDE directive.
+ //
+ // If a qfile contains an INCLUDE directive, and hadoopVer does
+ // not appear in the list of versions to include, then the qfile
+ // is skipped.
+ //
+ // If a qfile contains an EXCLUDE directive, and hadoopVer is
+ // listed in the list of versions to EXCLUDE, then the qfile is
+ // skipped.
+ //
+ // Otherwise, the qfile is included.
+
+ if (matcher.find()) {
+
+ String prefix = matcher.group(1);
+ if ("EX".equals(prefix)) {
+ excludeQuery = true;
+ } else {
+ includeQuery = true;
+ }
+
+ String versions = matcher.group(2);
+ for (String s : versions.split("\\,")) {
+ s = s.trim();
+ versionSet.add(s);
+ }
+ }
+
+ if (matcher.find()) {
+ //2nd match is not supposed to be there
+ String message = "QTestUtil: qfile " + fileName
+ + " contains more than one reference to (EX|IN)CLUDE_HADOOP_MAJOR_VERSIONS";
+ throw new UnsupportedOperationException(message);
+ }
+
+ if (excludeQuery && versionSet.contains(hadoopVer)) {
+ System.out.println("QTestUtil: " + fileName
+ + " EXCLUDE list contains Hadoop Version " + hadoopVer + ". Skipping...");
+ return true;
+ } else if (includeQuery && !versionSet.contains(hadoopVer)) {
+ System.out.println("QTestUtil: " + fileName
+ + " INCLUDE list does not contain Hadoop Version " + hadoopVer + ". Skipping...");
+ return true;
+ }
+ return false;
+ }
+
+ private boolean checkNeedJavaSpecificOutput(String fileName, String query) {
+ Pattern pattern = Pattern.compile("-- JAVA_VERSION_SPECIFIC_OUTPUT");
+ Matcher matcher = pattern.matcher(query);
+ if (matcher.find()) {
+ System.out.println("Test is flagged to generate Java version specific " +
+ "output. Since we are using Java version " + javaVersion +
+ ", we will generated Java " + javaVersion + " specific " +
+ "output file for query file " + fileName);
+ return true;
+ }
+
+ return false;
+ }
+
+ /**
+ * Get formatted Java version to include minor version, but
+ * exclude patch level.
+ *
+ * @return Java version formatted as major_version.minor_version
+ */
+ private static String getJavaVersion() {
+ String version = System.getProperty("java.version");
+ if (version == null) {
+ throw new NullPointerException("No java version could be determined " +
+ "from system properties");
+ }
+
+ // "java version" system property is formatted
+ // major_version.minor_version.patch_level.
+ // Find second dot, instead of last dot, to be safe
+ int pos = version.indexOf('.');
+ pos = version.indexOf('.', pos + 1);
+ return version.substring(0, pos);
+ }
+
+ /**
+ * Clear out any side effects of running tests
+ */
+ public void clearPostTestEffects() throws Exception {
+ setup.postTest(conf);
+ }
+
+ public void clearKeysCreatedInTests() {
+ if (hes == null) {
+ return;
+ }
+ try {
+ for (String keyAlias : hes.getKeys()) {
+ hes.deleteKey(keyAlias);
+ }
+ } catch (IOException e) {
+ LOG.error("Fail to clean the keys created in test due to the error", e);
+ }
+ }
+
+ public void clearUDFsCreatedDuringTests() throws Exception {
+ if (System.getenv(QTEST_LEAVE_FILES) != null) {
+ return;
+ }
+ // Delete functions created by the tests
+ // It is enough to remove functions from the default database, other databases are dropped
+ for (String udfName : db.getFunctions(DEFAULT_DATABASE_NAME, ".*")) {
+ if (!srcUDFs.contains(udfName)) {
+ db.dropFunction(DEFAULT_DATABASE_NAME, udfName);
+ }
+ }
+ }
+
+ /**
+ * Clear out any side effects of running tests
+ */
+ public void clearTablesCreatedDuringTests() throws Exception {
+ if (System.getenv(QTEST_LEAVE_FILES) != null) {
+ return;
+ }
+
+ conf.set("hive.metastore.filter.hook",
+ "org.apache.hadoop.hive.metastore.DefaultMetaStoreFilterHookImpl");
+ db = Hive.get(conf);
+
+ // First delete any MVs to avoid race conditions
+ for (String dbName : db.getAllDatabases()) {
+ SessionState.get().setCurrentDatabase(dbName);
+ for (String tblName : db.getAllTables()) {
+ Table tblObj = null;
+ try {
+ tblObj = db.getTable(tblName);
+ } catch (InvalidTableException e) {
+ LOG.warn("Trying to drop table " + e.getTableName() + ". But it does not exist.");
+ continue;
+ }
+ // only remove MVs first
+ if (!tblObj.isMaterializedView()) {
+ continue;
+ }
+ db.dropTable(dbName, tblName, true, true, fsType == FsType.encrypted_hdfs);
+ }
+ }
+
+ // Delete any tables other than the source tables
+ // and any databases other than the default database.
+ for (String dbName : db.getAllDatabases()) {
+ SessionState.get().setCurrentDatabase(dbName);
+ for (String tblName : db.getAllTables()) {
+ if (!DEFAULT_DATABASE_NAME.equals(dbName) || !srcTables.contains(tblName)) {
+ Table tblObj = null;
+ try {
+ tblObj = db.getTable(tblName);
+ } catch (InvalidTableException e) {
+ LOG.warn("Trying to drop table " + e.getTableName() + ". But it does not exist.");
+ continue;
+ }
+ // only remove MVs first
+ if (!tblObj.isMaterializedView()) {
+ continue;
+ }
+ db.dropTable(dbName, tblName, true, true, fsType == FsType.encrypted_hdfs);
+ }
+ }
+ if (!DEFAULT_DATABASE_NAME.equals(dbName)) {
+ // Drop cascade, functions dropped by cascade
+ db.dropDatabase(dbName, true, true, true);
+ }
+ }
+
+ // delete remaining directories for external tables (can affect stats for following tests)
+ try {
+ Path p = new Path(testWarehouse);
+ FileSystem fileSystem = p.getFileSystem(conf);
+ if (fileSystem.exists(p)) {
+ for (FileStatus status : fileSystem.listStatus(p)) {
+ if (status.isDirectory() && !srcTables.contains(status.getPath().getName())) {
+ fileSystem.delete(status.getPath(), true);
+ }
+ }
+ }
+ } catch (IllegalArgumentException e) {
+ // ignore.. provides invalid url sometimes intentionally
+ }
+ SessionState.get().setCurrentDatabase(DEFAULT_DATABASE_NAME);
+
+ List<String> roleNames = db.getAllRoleNames();
+ for (String roleName : roleNames) {
+ if (!"PUBLIC".equalsIgnoreCase(roleName) && !"ADMIN".equalsIgnoreCase(roleName)) {
+ db.dropRole(roleName);
+ }
+ }
+ }
+
+ /**
+ * Clear out any side effects of running tests
+ */
+ public void clearTestSideEffects() throws Exception {
+ if (System.getenv(QTEST_LEAVE_FILES) != null) {
+ return;
+ }
+
+ // Remove any cached results from the previous test.
+ QueryResultsCache.cleanupInstance();
+
+ // allocate and initialize a new conf since a test can
+ // modify conf by using 'set' commands
+ conf = new HiveConf(IDriver.class);
+ initConf();
+ initConfFromSetup();
+
+ // renew the metastore since the cluster type is unencrypted
+ db = Hive.get(conf); // propagate new conf to meta store
+
+ clearTablesCreatedDuringTests();
+ clearUDFsCreatedDuringTests();
+ clearKeysCreatedInTests();
+ }
+
+ protected void initConfFromSetup() throws Exception {
+ setup.preTest(conf);
+ }
+
+ public void cleanUp() throws Exception {
+ cleanUp(null);
+ }
+
+ public void cleanUp(String tname) throws Exception {
+ boolean canReuseSession = (tname == null) || !qNoSessionReuseQuerySet.contains(tname);
+ if(!isSessionStateStarted) {
+ startSessionState(canReuseSession);
+ }
+ if (System.getenv(QTEST_LEAVE_FILES) != null) {
+ return;
+ }
+
+ clearTablesCreatedDuringTests();
+ clearUDFsCreatedDuringTests();
+ clearKeysCreatedInTests();
+
+ File cleanupFile = new File(cleanupScript);
+ if (cleanupFile.isFile()) {
+ String cleanupCommands = readEntireFileIntoString(cleanupFile);
+ LOG.info("Cleanup (" + cleanupScript + "):\n" + cleanupCommands);
+ if(cliDriver == null) {
+ cliDriver = new CliDriver();
+ }
+ SessionState.get().getConf().setBoolean("hive.test.shutdown.phase", true);
+ int result = cliDriver.processLine(cleanupCommands);
+ if (result != 0) {
+ LOG.error("Failed during cleanup processLine with code={}. Ignoring", result);
+ // TODO Convert this to an Assert.fail once HIVE-14682 is fixed
+ }
+ SessionState.get().getConf().setBoolean("hive.test.shutdown.phase", false);
+ } else {
+ LOG.info("No cleanup script detected. Skipping.");
+ }
+
+ // delete any contents in the warehouse dir
+ Path p = new Path(testWarehouse);
+ FileSystem fs = p.getFileSystem(conf);
+
+ try {
+ FileStatus [] ls = fs.listStatus(p);
+ for (int i=0; (ls != null) && (i<ls.length); i++) {
+ fs.delete(ls[i].getPath(), true);
+ }
+ } catch (FileNotFoundException e) {
+ // Best effort
+ }
+
+ // TODO: Clean up all the other paths that are created.
+
+ FunctionRegistry.unregisterTemporaryUDF("test_udaf");
+ FunctionRegistry.unregisterTemporaryUDF("test_error");
+ }
+
+ protected void runCreateTableCmd(String createTableCmd) throws Exception {
+ int ecode = 0;
+ ecode = drv.run(createTableCmd).getResponseCode();
+ if (ecode != 0) {
+ throw new Exception("create table command: " + createTableCmd
+ + " failed with exit code= " + ecode);
+ }
+
+ return;
+ }
+
+ protected void runCmd(String cmd) throws Exception {
+ int ecode = 0;
+ ecode = drv.run(cmd).getResponseCode();
+ drv.close();
+ if (ecode != 0) {
+ throw new Exception("command: " + cmd
+ + " failed with exit code= " + ecode);
+ }
+ return;
+ }
+
+ public void createSources() throws Exception {
+ createSources(null);
+ }
+
+ public void createSources(String tname) throws Exception {
+ boolean canReuseSession = (tname == null) || !qNoSessionReuseQuerySet.contains(tname);
+ if(!isSessionStateStarted) {
+ startSessionState(canReuseSession);
+ }
+
+ if(cliDriver == null) {
+ cliDriver = new CliDriver();
+ }
+ cliDriver.processLine("set test.data.dir=" + testFiles + ";");
+ File scriptFile = new File(this.initScript);
+ if (!scriptFile.isFile()) {
+ LOG.info("No init script detected. Skipping");
+ return;
+ }
+ conf.setBoolean("hive.test.init.phase", true);
+
+ String initCommands = readEntireFileIntoString(scriptFile);
+ LOG.info("Initial setup (" + initScript + "):\n" + initCommands);
+
+ int result = cliDriver.processLine(initCommands);
+ LOG.info("Result from cliDrriver.processLine in createSources=" + result);
+ if (result != 0) {
+ Assert.fail("Failed during createSources processLine with code=" + result);
+ }
+
+ conf.setBoolean("hive.test.init.phase", false);
+ }
+
+ public void init() throws Exception {
+
+ // Create remote dirs once.
+ if (mr != null) {
+ createRemoteDirs();
+ }
+
+ // Create views registry
+ HiveMaterializedViewsRegistry.get().init();
+
+ testWarehouse = conf.getVar(HiveConf.ConfVars.METASTOREWAREHOUSE);
+ String execEngine = conf.get("hive.execution.engine");
+ conf.set("hive.execution.engine", "mr");
+ SessionState.start(conf);
+ conf.set("hive.execution.engine", execEngine);
+ db = Hive.get(conf);
+ drv = DriverFactory.newDriver(conf);
+ pd = new ParseDriver();
+ sem = new SemanticAnalyzer(queryState);
+ }
+
+ public void init(String tname) throws Exception {
+ cleanUp(tname);
+ createSources(tname);
+ cliDriver.processCmd("set hive.cli.print.header=true;");
+ }
+
+ public void cliInit(String tname) throws Exception {
+ cliInit(tname, true);
+ }
+
+ public String cliInit(String tname, boolean recreate) throws Exception {
+ if (recreate) {
+ cleanUp(tname);
+ createSources(tname);
+ }
+
+ HiveConf.setVar(conf, HiveConf.ConfVars.HIVE_AUTHENTICATOR_MANAGER,
+ "org.apache.hadoop.hive.ql.security.DummyAuthenticator");
+ Utilities.clearWorkMap(conf);
+ CliSessionState ss = new CliSessionState(conf);
+ assert ss != null;
+ ss.in = System.in;
+
+ String outFileExtension = getOutFileExtension(tname);
+ String stdoutName = null;
+ if (outDir != null) {
+ // TODO: why is this needed?
+ File qf = new File(outDir, tname);
+ stdoutName = qf.getName().concat(outFileExtension);
+ } else {
+ stdoutName = tname + outFileExtension;
+ }
+
+ File outf = new File(logDir, stdoutName);
+ OutputStream fo = new BufferedOutputStream(new FileOutputStream(outf));
+ if (qSortQuerySet.contains(tname)) {
+ ss.out = new SortPrintStream(fo, "UTF-8");
+ } else if (qHashQuerySet.contains(tname)) {
+ ss.out = new DigestPrintStream(fo, "UTF-8");
+ } else if (qSortNHashQuerySet.contains(tname)) {
+ ss.out = new SortAndDigestPrintStream(fo, "UTF-8");
+ } else {
+ ss.out = new PrintStream(fo, true, "UTF-8");
+ }
+ ss.err = new CachingPrintStream(fo, true, "UTF-8");
+ ss.setIsSilent(true);
+ SessionState oldSs = SessionState.get();
+
+ boolean canReuseSession = !qNoSessionReuseQuerySet.contains(tname);
+ if (oldSs != null && canReuseSession && clusterType.getCoreClusterType() == CoreClusterType.TEZ) {
+ // Copy the tezSessionState from the old CliSessionState.
+ TezSessionState tezSessionState = oldSs.getTezSession();
+ oldSs.setTezSession(null);
+ ss.setTezSession(tezSessionState);
+ oldSs.close();
+ }
+
+ if (oldSs != null && clusterType.getCoreClusterType() == CoreClusterType.SPARK) {
+ sparkSession = oldSs.getSparkSession();
+ ss.setSparkSession(sparkSession);
+ oldSs.setSparkSession(null);
+ oldSs.close();
+ }
+
+ if (oldSs != null && oldSs.out != null && oldSs.out != System.out) {
+ oldSs.out.close();
+ }
+ if (oldSs != null) {
+ oldSs.close();
+ }
+ SessionState.start(ss);
+
+ cliDriver = new CliDriver();
+
+ if (tname.equals("init_file.q")) {
+ ss.initFiles.add(HIVE_ROOT + "/data/scripts/test_init_file.sql");
+ }
+ cliDriver.processInitFiles(ss);
+
+ return outf.getAbsolutePath();
+ }
+
+ private CliSessionState startSessionState(boolean canReuseSession)
+ throws IOException {
+
+ HiveConf.setVar(conf, HiveConf.ConfVars.HIVE_AUTHENTICATOR_MANAGER,
+ "org.apache.hadoop.hive.ql.security.DummyAuthenticator");
+
+ String execEngine = conf.get("hive.execution.engine");
+ conf.set("hive.execution.engine", "mr");
+ CliSessionState ss = new CliSessionState(conf);
+ assert ss != null;
+ ss.in = System.in;
+ ss.out = System.out;
+ ss.err = System.out;
+
+ SessionState oldSs = SessionState.get();
+ if (oldSs != null && canReuseSession && clusterType.getCoreClusterType() == CoreClusterType.TEZ) {
+ // Copy the tezSessionState from the old CliSessionState.
+ TezSessionState tezSessionState = oldSs.getTezSession();
+ ss.setTezSession(tezSessionState);
+ oldSs.setTezSession(null);
+ oldSs.close();
+ }
+
+ if (oldSs != null && clusterType.getCoreClusterType() == CoreClusterType.SPARK) {
+ sparkSession = oldSs.getSparkSession();
+ ss.setSparkSession(sparkSession);
+ oldSs.setSparkSession(null);
+ oldSs.close();
+ }
+ if (oldSs != null && oldSs.out != null && oldSs.out != System.out) {
+ oldSs.out.close();
+ }
+ if (oldSs != null) {
+ oldSs.close();
+ }
+ SessionState.start(ss);
+
+ isSessionStateStarted = true;
+
+ conf.set("hive.execution.engine", execEngine);
+ return ss;
+ }
+
+ public int executeAdhocCommand(String q) {
+ if (!q.contains(";")) {
+ return -1;
+ }
+
+ String q1 = q.split(";")[0] + ";";
+
+ LOG.debug("Executing " + q1);
+ return cliDriver.processLine(q1);
+ }
+
+ public int executeOne(String tname) {
+ String q = qMap.get(tname);
+
+ if (q.indexOf(";") == -1) {
+ return -1;
+ }
+
+ String q1 = q.substring(0, q.indexOf(";") + 1);
+ String qrest = q.substring(q.indexOf(";") + 1);
+ qMap.put(tname, qrest);
+
+ System.out.println("Executing " + q1);
+ return cliDriver.processLine(q1);
+ }
+
+ public int execute(String tname) {
+ return drv.run(qMap.get(tname)).getResponseCode();
+ }
+
+ public int executeClient(String tname1, String tname2) {
+ String commands = getCommand(tname1) + CRLF + getCommand(tname2);
+ return executeClientInternal(commands);
+ }
+
+ public int executeClient(String tname) {
+ return executeClientInternal(getCommand(tname));
+ }
+
+ private int executeClientInternal(String commands) {
+ List<String> cmds = CliDriver.splitSemiColon(commands);
+ int rc = 0;
+
+ String command = "";
+ for (String oneCmd : cmds) {
+ if (StringUtils.endsWith(oneCmd, "\\")) {
+ command += StringUtils.chop(oneCmd) + "\\;";
+ continue;
+ } else {
+ if (isHiveCommand(oneCmd)) {
+ command = oneCmd;
+ } else {
+ command += oneCmd;
+ }
+ }
+ if (StringUtils.isBlank(command)) {
+ continue;
+ }
+
+ if (isCommandUsedForTesting(command)) {
+ rc = executeTestCommand(command);
+ } else {
+ rc = cliDriver.processLine(command);
+ }
+
+ if (rc != 0 && !ignoreErrors()) {
+ break;
+ }
+ command = "";
+ }
+ if (rc == 0 && SessionState.get() != null) {
+ SessionState.get().setLastCommand(null); // reset
+ }
+ return rc;
+ }
+
+ /**
+ * This allows a .q file to continue executing after a statement runs into an error which is convenient
+ * if you want to use another hive cmd after the failure to sanity check the state of the system.
+ */
+ private boolean ignoreErrors() {
+ return conf.getBoolVar(HiveConf.ConfVars.CLIIGNOREERRORS);
+ }
+
+ private boolean isHiveCommand(String command) {
+ String[] cmd = command.trim().split("\\s+");
+ if (HiveCommand.find(cmd) != null) {
+ return true;
+ } else if (HiveCommand.find(cmd, HiveCommand.ONLY_FOR_TESTING) != null) {
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ private int executeTestCommand(final String command) {
+ String commandName = command.trim().split("\\s+")[0];
+ String commandArgs = command.trim().substring(commandName.length());
+
+ if (commandArgs.endsWith(";")) {
+ commandArgs = StringUtils.chop(commandArgs);
+ }
+
+ //replace ${hiveconf:hive.metastore.warehouse.dir} with actual dir if existed.
+ //we only want the absolute path, so remove the header, such as hdfs://localhost:57145
+ String wareHouseDir = SessionState.get().getConf().getVar(ConfVars.METASTOREWAREHOUSE)
+ .replaceAll("^[a-zA-Z]+://.*?:\\d+", "");
+ commandArgs = commandArgs.replaceAll("\\$\\{hiveconf:hive\\.metastore\\.warehouse\\.dir\\}",
+ wareHouseDir);
+
+ if (SessionState.get() != null) {
+ SessionState.get().setLastCommand(commandName + " " + commandArgs.trim());
+ }
+
+ enableTestOnlyCmd(SessionState.get().getConf());
+
+ try {
+ CommandProcessor proc = getTestCommand(commandName);
+ if (proc != null) {
+ CommandProcessorResponse response = proc.run(commandArgs.trim());
+
+ int rc = response.getResponseCode();
+ if (rc != 0) {
+ SessionState.getConsole().printError(response.toString(), response.getException() != null ?
+ Throwables.getStackTraceAsString(response.getException()) : "");
+ }
+
+ return rc;
+ } else {
+ throw new RuntimeException("Could not get CommandProcessor for command: " + commandName);
+ }
+ } catch (Exception e) {
+ throw new RuntimeException("Could not execute test command", e);
+ }
+ }
+
+ private CommandProcessor getTestCommand(final String commandName) throws SQLException {
+ HiveCommand testCommand = HiveCommand.find(new String[]{commandName}, HiveCommand.ONLY_FOR_TESTING);
+
+ if (testCommand == null) {
+ return null;
+ }
+
+ return CommandProcessorFactory
+ .getForHiveCommandInternal(new String[]{commandName}, SessionState.get().getConf(),
+ testCommand.isOnlyForTesting());
+ }
+
+ private void enableTestOnlyCmd(HiveConf conf){
+ StringBuilder securityCMDs = new StringBuilder(conf.getVar(HiveConf.ConfVars.HIVE_SECURITY_COMMAND_WHITELIST));
+ for(String c : testOnlyCommands){
+ securityCMDs.append(",");
+ securityCMDs.append(c);
+ }
+ conf.set(HiveConf.ConfVars.HIVE_SECURITY_COMMAND_WHITELIST.toString(), securityCMDs.toString());
+ }
+
+ private boolean isCommandUsedForTesting(final String command) {
+ String commandName = command.trim().split("\\s+")[0];
+ HiveCommand testCommand = HiveCommand.find(new String[]{commandName}, HiveCommand.ONLY_FOR_TESTING);
+ return testCommand != null;
+ }
+
+ private String getCommand(String tname) {
+ String commands = qMap.get(tname);
+ StringBuilder newCommands = new StringBuilder(commands.length());
+ int lastMatchEnd = 0;
+ Matcher commentMatcher = Pattern.compile("^--.*$", Pattern.MULTILINE).matcher(commands);
+ // remove the comments
+ while (commentMatcher.find()) {
+ newCommands.append(commands.substring(lastMatchEnd, commentMatcher.start()));
+ lastMatchEnd = commentMatcher.end();
+ }
+ newCommands.append(commands.substring(lastMatchEnd, commands.length()));
+ commands = newCommands.toString();
+ return commands;
+ }
+
+ public boolean shouldBeSkipped(String tname) {
+ return qSkipSet.contains(tname);
+ }
+
+ private String getOutFileExtension(String fname) {
+ String outFileExtension = ".out";
+ if (qJavaVersionSpecificOutput.contains(fname)) {
+ outFileExtension = ".java" + javaVersion + ".out";
+ }
+
+ return outFileExtension;
+ }
+
+ public void convertSequenceFileToTextFile() throws Exception {
+ // Create an instance of hive in order to create the tables
+ testWarehouse = conf.getVar(HiveConf.ConfVars.METASTOREWAREHOUSE);
+ db = Hive.get(conf);
+
+ // Move all data from dest4_sequencefile to dest4
+ drv
+ .run("FROM dest4_sequencefile INSERT OVERWRITE TABLE dest4 SELECT dest4_sequencefile.*");
+
+ // Drop dest4_sequencefile
+ db.dropTable(Warehouse.DEFAULT_DATABASE_NAME, "dest4_sequencefile",
+ true, true);
+ }
+
+ public QTestProcessExecResult checkNegativeResults(String tname, Exception e) throws Exception {
+
+ String outFileExtension = getOutFileExtension(tname);
+
+ File qf = new File(outDir, tname);
+ String expf = outPath(outDir.toString(), tname.concat(outFileExtension));
+
+ File outf = null;
+ outf = new File(logDir);
+ outf = new File(outf, qf.getName().concat(outFileExtension));
+
+ FileWriter outfd = new FileWriter(outf);
+ if (e instanceof ParseException) {
+ outfd.write("Parse Error: ");
+ } else if (e instanceof SemanticException) {
+ outfd.write("Semantic Exception: \n");
+ } else {
+ throw e;
+ }
+
+ outfd.write(e.getMessage());
+ outfd.close();
+
+ QTestProcessExecResult result = executeDiffCommand(outf.getPath(), expf, false,
+ qSortSet.contains(qf.getName()));
+ if (overWrite) {
+ overwriteResults(outf.getPath(), expf);
+ return QTestProcessExecResult.createWithoutOutput(0);
+ }
+
+ return result;
+ }
+
+ public QTestProcessExecResult checkParseResults(String tname, ASTNode tree) throws Exception {
+
+ if (tree != null) {
+ String outFileExtension = getOutFileExtension(tname);
+
+ File parseDir = new File(outDir, "parse");
+ String expf = outPath(parseDir.toString(), tname.concat(outFileExtension));
+
+ File outf = null;
+ outf = new File(logDir);
+ outf = new File(outf, tname.concat(outFileExtension));
+
+ FileWriter outfd = new FileWriter(outf);
+ outfd.write(tree.toStringTree());
+ outfd.close();
+
+ QTestProcessExecResult exitVal = executeDiffCommand(outf.getPath(), expf, false, false);
+
+ if (overWrite) {
+ overwriteResults(outf.getPath(), expf);
+ return QTestProcessExecResult.createWithoutOutput(0);
+ }
+
+ return exitVal;
+ } else {
+ throw new Exception("Parse tree is null");
+ }
+ }
+
+ /**
+ * Given the current configurations (e.g., hadoop version and execution mode), return
+ * the correct file name to compare with the current test run output.
+ * @param outDir The directory where the reference log files are stored.
+ * @param testName The test file name (terminated by ".out").
+ * @return The file name appended with the configuration values if it exists.
+ */
+ public String outPath(String outDir, String testName) {
+ String ret = (new File(outDir, testName)).getPath();
+ // List of configurations. Currently the list consists of hadoop version and execution mode only
+ List<String> configs = new ArrayList<String>();
+ configs.add(this.clusterType.toString());
+ configs.add(this.hadoopVer);
+
+ Deque<String> stack = new LinkedList<String>();
+ StringBuilder sb = new StringBuilder();
+ sb.append(testName);
+ stack.push(sb.toString());
+
+ // example file names are input1.q.out_mr_0.17 or input2.q.out_0.17
+ for (String s: configs) {
+ sb.append('_');
+ sb.append(s);
+ stack.push(sb.toString());
+ }
+ while (stack.size() > 0) {
+ String fileName = stack.pop();
+ File f = new File(outDir, fileName);
+ if (f.exists()) {
+ ret = f.getPath();
+ break;
+ }
+ }
+ return ret;
+ }
+
+ private Pattern[] toPattern(String[] patternStrs) {
+ Pattern[] patterns = new Pattern[patternStrs.length];
+ for (int i = 0; i < patternStrs.length; i++) {
+ patterns[i] = Pattern.compile(patternStrs[i]);
+ }
+ return patterns;
+ }
+
+ private void maskPatterns(Pattern[] patterns, String fname) throws Exception {
+ String maskPattern = "#### A masked pattern was here ####";
+ String partialMaskPattern = "#### A PARTIAL masked pattern was here ####";
+
+ String line;
+ BufferedReader in;
+ BufferedWriter out;
+
+ File file = new File(fname);
+ File fileOrig = new File(fname + ".orig");
+ FileUtils.copyFile(file, fileOrig);
+
+ in = new BufferedReader(new InputStreamReader(new FileInputStream(fileOrig), "UTF-8"));
+ out = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(file), "UTF-8"));
+
+ boolean lastWasMasked = false;
+ boolean partialMaskWasMatched = false;
+ Matcher matcher;
+ while (null != (line = in.readLine())) {
+ if (fsType == FsType.encrypted_hdfs) {
+ for (Pattern pattern : partialReservedPlanMask) {
+ matcher = pattern.matcher(line);
+ if (matcher.find()) {
+ line = partialMaskPattern + " " + matcher.group(0);
+ partialMaskWasMatched = true;
+ break;
+ }
+ }
+ }
+ else {
+ for (PatternReplacementPair prp : partialPlanMask) {
+ matcher = prp.pattern.matcher(line);
+ if (matcher.find()) {
+ line = line.replaceAll(prp.pattern.pattern(), prp.replacement);
+ partialMaskWasMatched = true;
+ }
+ }
+ }
+
+ if (!partialMaskWasMatched) {
+ for (Pair<Pattern, String> pair : patternsWithMaskComments) {
+ Pattern pattern = pair.getLeft();
+ String maskComment = pair.getRight();
+
+ matcher = pattern.matcher(line);
+ if (matcher.find()) {
+ line = matcher.replaceAll(maskComment);
+ partialMaskWasMatched = true;
+ break;
+ }
+ }
+
+ for (Pattern pattern : patterns) {
+ line = pattern.matcher(line).replaceAll(maskPattern);
+ }
+ }
+
+ if (line.equals(maskPattern)) {
+ // We're folding multiple masked lines into one.
+ if (!lastWasMasked) {
+ out.write(line);
+ out.write("\n");
+ lastWasMasked = true;
+ partialMaskWasMatched = false;
+ }
+ } else {
+ out.write(line);
+ out.write("\n");
+ lastWasMasked = false;
+ partialMaskWasMatched = false;
+ }
+ }
+
+ in.close();
+ out.close();
+ }
+
+ private final Pattern[] planMask = toPattern(new String[] {
+ ".*file:.*",
+ ".*pfile:.*",
+ ".*/tmp/.*",
+ ".*invalidscheme:.*",
+ ".*lastUpdateTime.*",
+ ".*lastAccessTime.*",
+ ".*lastModifiedTime.*",
+ ".*[Oo]wner.*",
+ ".*CreateTime.*",
+ ".*LastAccessTime.*",
+ ".*Location.*",
+ ".*LOCATION '.*",
+ ".*transient_lastDdlTime.*",
+ ".*last_modified_.*",
+ ".*at org.*",
+ ".*at sun.*",
+ ".*at java.*",
+ ".*at junit.*",
+ ".*Caused by:.*",
+ ".*LOCK_QUERYID:.*",
+ ".*LOCK_TIME:.*",
+ ".*grantTime.*",
+ ".*[.][.][.] [0-9]* more.*",
+ ".*job_[0-9_]*.*",
+ ".*job_local[0-9_]*.*",
+ ".*USING 'java -cp.*",
+ "^Deleted.*",
+ ".*DagName:.*",
+ ".*DagId:.*",
+ ".*Input:.*/data/files/.*",
+ ".*Output:.*/data/files/.*",
+ ".*total number of created files now is.*",
+ ".*.hive-staging.*",
+ ".*Warning.*",
+ "pk_-?[0-9]*_[0-9]*_[0-9]*",
+ "fk_-?[0-9]*_[0-9]*_[0-9]*",
+ "uk_-?[0-9]*_[0-9]*_[0-9]*",
+ "nn_-?[0-9]*_[0-9]*_[0-9]*",
+ ".*at com\\.sun\\.proxy.*",
+ ".*at com\\.jolbox.*",
+ ".*at com\\.zaxxer.*",
+ "org\\.apache\\.hadoop\\.hive\\.metastore\\.model\\.MConstraint@([0-9]|[a-z])*",
+ "^Repair: Added partition to metastore.*"
+ });
+
+ private final Pattern[] partialReservedPlanMask = toPattern(new String[] {
+ "data/warehouse/(.*?/)+\\.hive-staging" // the directory might be db/table/partition
+ //TODO: add more expected test result here
+ });
+ /**
+ * Pattern to match and (partial) replacement text.
+ * For example, {"transaction":76,"bucketid":8249877}. We just want to mask 76 but a regex that
+ * matches just 76 will match a lot of other things.
+ */
+ private final static class PatternReplacementPair {
+ private final Pattern pattern;
+ private final String replacement;
+ PatternReplacementPair(Pattern p, String r) {
+ pattern = p;
+ replacement = r;
+ }
+ }
+ private final PatternReplacementPair[] partialPlanMask;
+ {
+ ArrayList<PatternReplacementPair> ppm = new ArrayList<>();
+ ppm.add(new PatternReplacementPair(Pattern.compile("\\{\"transactionid\":[1-9][0-9]*,\"bucketid\":"),
+ "{\"transactionid\":### Masked txnid ###,\"bucketid\":"));
+
+ ppm.add(new PatternReplacementPair(Pattern.compile("attempt_[0-9]+"), "attempt_#ID#"));
+ ppm.add(new PatternReplacementPair(Pattern.compile("vertex_[0-9_]+"), "vertex_#ID#"));
+ ppm.add(new PatternReplacementPair(Pattern.compile("task_[0-9_]+"), "task_#ID#"));
+ partialPlanMask = ppm.toArray(new PatternReplacementPair[ppm.size()]);
+ }
+ /* This list may be modified by specific cli drivers to mask strings that change on every test */
+ private final List<Pair<Pattern, String>> patternsWithMaskComments =
+ new ArrayList<Pair<Pattern, String>>() {
+ {
+ add(toPatternPair("(pblob|s3.?|swift|wasb.?).*hive-staging.*",
+ "### BLOBSTORE_STAGING_PATH ###"));
+ add(toPatternPair(PATH_HDFS_WITH_DATE_USER_GROUP_REGEX,
+ "### USER ### ### GROUP ###$3$4 ### HDFS DATE ### $6### HDFS PATH ###"));
+ add(toPatternPair(PATH_HDFS_REGEX, "$1### HDFS PATH ###"));
+ }
+ };
+
+ private Pair<Pattern, String> toPatternPair(String patternStr, String maskComment) {
+ return ImmutablePair.of(Pattern.compile(patternStr), maskComment);
+ }
+
+ public void addPatternWithMaskComment(String patternStr, String maskComment) {
+ patternsWithMaskComments.add(toPatternPair(patternStr, maskComment));
+ }
+
+ public QTestProcessExecResult checkCliDriverResults(String tname) throws Exception {
+ assert(qMap.containsKey(tname));
+
+ String outFileExtension = getOutFileExtension(tname);
+ String outFileName = outPath(outDir, tname + outFileExtension);
+
+ File f = new File(logDir, tname + outFileExtension);
+
+ maskPatterns(planMask, f.getPath());
+ QTestProcessExecResult exitVal = executeDiffCommand(f.getPath(),
+ outFileName, false,
+ qSortSet.contains(tname));
+
+ if (overWrite) {
+ overwriteResults(f.getPath(), outFileName);
+ return QTestProcessExecResult.createWithoutOutput(0);
+ }
+
+ return exitVal;
+ }
+
+
+ public QTestProcessExecResult checkCompareCliDriverResults(String tname, List<String> outputs)
+ throws Exception {
+ assert outputs.size() > 1;
+ maskPatterns(planMask, outputs.get(0));
+ for (int i = 1; i < outputs.size(); ++i) {
+ maskPatterns(planMask, outputs.get(i));
+ QTestProcessExecResult result = executeDiffCommand(
+ outputs.get(i - 1), outputs.get(i), false, qSortSet.contains(tname));
+ if (result.getReturnCode() != 0) {
+ System.out.println("Files don't match: " + outputs.get(i - 1) + " and " + outputs.get(i));
+ return result;
+ }
+ }
+ return QTestProcessExecResult.createWithoutOutput(0);
+ }
+
+ private static void overwriteResults(String inFileName, String outFileName) throws Exception {
+ // This method can be replaced with Files.copy(source, target, REPLACE_EXISTING)
+ // once Hive uses JAVA 7.
+ System.out.println("Overwriting results " + inFileName + " to " + outFileName);
+ int result = executeCmd(new String[]{
+ "cp",
+ getQuotedString(inFileName),
+ getQuotedString(outFileName)
+ }).getReturnCode();
+ if (result != 0) {
+ throw new IllegalStateException("Unexpected error while overwriting " +
+ inFileName + " with " + outFileName);
+ }
+ }
+
+ private static QTestProcessExecResult executeDiffCommand(String inFileName,
+ String outFileName,
+ boolean ignoreWhiteSpace,
+ boolean sortResults
+ ) throws Exception {
+
+ QTestProcessExecResult result;
+
+ if (sortResults) {
+ // sort will try to open the output file in write mode on windows. We need to
+ // close it first.
+ SessionState ss = SessionState.get();
+ if (ss != null && ss.out != null && ss.out != System.out) {
+ ss.out.close();
+ }
+
+ String inSorted = inFileName + SORT_SUFFIX;
+ String outSorted = outFileName + SORT_SUFFIX;
+
+ sortFiles(inFileName, inSorted);
+ sortFiles(outFileName, outSorted);
+
+ inFileName = inSorted;
+ outFileName = outSorted;
+ }
+
+ ArrayList<String> diffCommandArgs = new ArrayList<String>();
+ diffCommandArgs.add("diff");
+
+ // Text file comparison
+ diffCommandArgs.add("-a");
+
+ // Ignore changes in the amount of white space
+ if (ignoreWhiteSpace) {
+ diffCommandArgs.add("-b");
+ }
+
+ // Add files to compare to the arguments list
+ diffCommandArgs.add(getQuotedString(inFileName));
+ diffCommandArgs.add(getQuotedString(outFileName));
+
+ result = executeCmd(diffCommandArgs);
+
+ if (sortResults) {
+ new File(inFileName).delete();
+ new File(outFileName).delete();
+ }
+
+ return result;
+ }
+
+ private static void sortFiles(String in, String out) throws Exception {
+ int result = executeCmd(new String[]{
+ "sort",
+ getQuotedString(in),
+ }, out, null).getReturnCode();
+ if (result != 0) {
+ throw new IllegalStateException("Unexpected error while sorting " + in);
+ }
+ }
+
+ private static QTestProcessExecResult executeCmd(Collection<String> args) throws Exception {
+ return executeCmd(args, null, null);
+ }
+
+ private static QTestProcessExecResult executeCmd(String[] args) throws Exception {
+ return executeCmd(args, null, null);
+ }
+
+ private static QTestProcessExecResult executeCmd(Collection<String> args, String outFile,
+ String errFile) throws Exception {
+ String[] cmdArray = args.toArray(new String[args.size()]);
+ return executeCmd(cmdArray, outFile, errFile);
+ }
+
+ private static QTestProcessExecResult executeCmd(String[] args, String outFile,
+ String errFile) throws Exception {
+ System.out.println("Running: " + org.apache.commons.lang.StringUtils.join(args, ' '));
+
+ PrintStream out = outFile == null ?
+ SessionState.getConsole().getChildOutStream() :
+ new PrintStream(new FileOutputStream(outFile), true, "UTF-8");
+ PrintStream err = errFile == null ?
+ SessionState.getConsole().getChildErrStream() :
+ new PrintStream(new FileOutputStream(errFile), true, "UTF-8");
+
+ Process executor = Runtime.getRuntime().exec(args);
+
+ ByteArrayOutputStream bos = new ByteArrayOutputStream();
+ PrintStream str = new PrintStream(bos, true, "UTF-8");
+
+ StreamPrinter errPrinter = new StreamPrinter(executor.getErrorStream(), null, err);
+ StreamPrinter outPrinter = new StreamPrinter(executor.getInputStream(), null, out, str);
+
+ outPrinter.start();
+ errPrinter.start();
+
+ int result = executor.waitFor();
+
+ outPrinter.join();
+ errPrinter.join();
+
+ if (outFile != null) {
+ out.close();
+ }
+
+ if (errFile != null) {
+ err.close();
+ }
+
+ return QTestProcessExecResult.
+ create(result, new String(bos.toByteArray(), StandardCharsets.UTF_8));
+ }
+
+ private static String getQuotedString(String str){
+ return str;
+ }
+
+ public ASTNode parseQuery(String tname) throws Exception {
+ return pd.parse(qMap.get(tname));
+ }
+
+ public void resetParser() throws SemanticException {
+ pd = new ParseDriver();
+ queryState = new QueryState.Builder().withHiveConf(conf).build();
+ sem = new SemanticAnalyzer(queryState);
+ }
+
+
+ public List<Task<? extends Serializable>> analyzeAST(ASTNode ast) throws Exception {
+
+ // Do semantic analysis and plan generation
+ Context ctx = new Context(conf);
+ while ((ast.getToken() == null) && (ast.getChildCount() > 0)) {
+ ast = (ASTNode) ast.getChild(0);
+ }
+ sem.getOutputs().clear();
+ sem.getInputs().clear();
+ sem.analyze(ast, ctx);
+ ctx.clear();
+ return sem.getRootTasks();
+ }
+
+ public TreeMap<String, String> getQMap() {
+ return qMap;
+ }
+
+ /**
+ * QTestSetup defines test fixtures which are reused across testcases,
+ * and are needed before any test can be run
+ */
+ public static class QTestSetup
+ {
+ private MiniZooKeeperCluster zooKeeperCluster = null;
+ private int zkPort;
+ private ZooKeeper zooKeeper;
+
+ public QTestSetup() {
+ }
+
+ public void preTest(HiveConf conf) throws Exception {
+
+ if (zooKeeperCluster == null) {
+ //create temp dir
+ String tmpBaseDir = System.getProperty(TEST_TMP_DIR_PROPERTY);
+ File tmpDir = Utilities.createTempDir(tmpBaseDir);
+
+ zooKeeperCluster = new MiniZooKeeperCluster();
+ zkPort = zooKeeperCluster.startup(tmpDir);
+ }
+
+ if (zooKeeper != null) {
+ zooKeeper.close();
+ }
+
+ int sessionTimeout = (int) conf.getTimeVar(HiveConf.ConfVars.HIVE_ZOOKEEPER_SESSION_TIMEOUT, TimeUnit.MILLISECONDS);
+ zooKeeper = new ZooKeeper("localhost:" + zkPort, sessionTimeout, new Watcher() {
+ @Override
+ public void process(WatchedEvent arg0) {
+ }
+ });
+
+ String zkServer = "localhost";
+ conf.set("hive.zookeeper.quorum", zkServer);
+ conf.set("hive.zookeeper.client.port", "" + zkPort);
+ }
+
+ public void postTest(HiveConf conf) throws Exception {
+ if (zooKeeperCluster == null) {
+ return;
+ }
+
+ if (zooKeeper != null) {
+ zooKeeper.close();
+ }
+
+ ZooKeeperHiveLockManager.releaseAllLocks(conf);
+ }
+
+ public void tearDown() throws Exception {
+ CuratorFrameworkSingleton.closeAndReleaseInstance();
+
+ if (zooKeeperCluster != null) {
+ zooKeeperCluster.shutdown();
+ zooKeeperCluster = null;
+ }
+ }
+ }
+
+ /**
+ * QTRunner: Runnable class for running a a single query file.
+ *
+ **/
+ public static class QTRunner implements Runnable {
+ private final QTestUtil qt;
+ private final String fname;
+
+ public QTRunner(QTestUtil qt, String fname) {
+ this.qt = qt;
+ this.fname = fname;
+ }
+
+ @Override
+ public void run() {
+ try {
+ // assumption is that environment has already been cleaned once globally
+ // hence each thread does not call cleanUp() and createSources() again
+ qt.cliInit(fname, false);
+ qt.executeClient(fname);
+ } catch (Throwable e) {
+ System.err.println("Query file " + fname + " failed with exception "
+ + e.getMessage());
+ e.printStackTrace();
+ outputTestFailureHelpMessage();
+ }
+ }
+ }
+
+ /**
+ * Setup to execute a set of query files. Uses QTestUtil to do so.
+ *
+ * @param qfiles
+ * array of input query files containing arbitrary number of hive
+ * queries
+ * @param resDir
+ * output directory
+ * @param logDir
+ * log directory
+ * @return one QTestUtil for each query file
+ */
+ public static QTestUtil[] queryListRunnerSetup(File[] qfiles, String resDir,
+ String logDir, String initScript, String cleanupScript) throws Exception
+ {
+ QTestUtil[] qt = new QTestUtil[qfiles.length];
+ for (int i = 0; i < qfiles.length; i++) {
+ qt[i] = new QTestUtil(resDir, logDir, MiniClusterType.none, null, "0.20",
+ initScript == null ? defaultInitScript : initScript,
+ cleanupScript == null ? defaultCleanupScript : cleanupScript, false);
+ qt[i].addFile(qfiles[i]);
+ qt[i].clearTestSideEffects();
+ }
+
+ return qt;
+ }
+
+ /**
+ * Executes a set of query files in sequence.
+ *
+ * @param qfiles
+ * array of input query files containing arbitrary number of hive
+ * queries
+ * @param qt
+ * array of QTestUtils, one per qfile
+ * @return true if all queries passed, false otw
+ */
+ public static boolean queryListRunnerSingleThreaded(File[] qfiles, QTestUtil[] qt)
+ throws Exception
+ {
+ boolean failed = false;
+ qt[0].cleanUp();
+ qt[0].createSources();
+ for (int i = 0; i < qfiles.length && !failed; i++) {
+ qt[i].clearTestSideEffects();
+ qt[i].cliInit(qfiles[i].getName(), false);
+ qt[i].executeClient(qfiles[i].getName());
+ QTestProcessExecResult result = qt[i].checkCliDriverResults(qfiles[i].getName());
+ if (result.getReturnCode() != 0) {
+ failed = true;
+ StringBuilder builder = new StringBuilder();
+ builder.append("Test ")
+ .append(qfiles[i].getName())
+ .append(" results check failed with error code ")
+ .append(result.getReturnCode());
+ if (Strings.isNotEmpty(result.getCapturedOutput())) {
+ builder.append(" and diff value ").append(result.getCapturedOutput());
+ }
+ System.err.println(builder.toString());
+ outputTestFailureHelpMessage();
+ }
+ qt[i].clearPostTestEffects();
+ }
+ return (!failed);
+ }
+
+ /**
+ * Executes a set of query files parallel.
+ *
+ * Each query file is run in a separate thread. The caller has to arrange
+ * that different query files do not collide (in terms of destination tables)
+ *
+ * @param qfiles
+ * array of input query files containing arbitrary number of hive
+ * queries
+ * @param qt
+ * array of QTestUtils, one per qfile
+ * @return true if all queries passed, false otw
+ *
+ */
+ public static boolean queryListRunnerMultiThreaded(File[] qfiles, QTestUtil[] qt)
+ throws Exception
+ {
+ boolean failed = false;
+
+ // in multithreaded mode - do cleanup/initialization just once
+
+ qt[0].cleanUp();
+ qt[0].createSources();
+ qt[0].clearTestSideEffects();
+
+ QTRunner[] qtRunners = new QTRunner[qfiles.length];
+ Thread[] qtThread = new Thread[qfiles.length];
+
+ for (int i = 0; i < qfiles.length; i++) {
+ qtRunners[i] = new QTRunner(qt[i], qfiles[i].getName());
+ qtThread[i] = new Thread(qtRunners[i]);
+ }
+
+ for (int i = 0; i < qfiles.length; i++) {
+ qtThread[i].start();
+ }
+
+ for (int i = 0; i < qfiles.length; i++) {
+ qtThread[i].join();
+ QTestProcessExecResult result = qt[i].checkCliDriverResults(qfiles[i].getName());
+ if (result.getReturnCode() != 0) {
+ failed = true;
+ StringBuilder builder = new StringBuilder();
+ builder.append("Test ")
+ .append(qfiles[i].getName())
+ .append(" results check failed with error code ")
+ .append(result.getReturnCode());
+ if (Strings.isNotEmpty(result.getCapturedOutput())) {
+ builder.append(" and diff value ").append(result.getCapturedOutput());
+ }
+ System.err.println(builder.toString());
+ outputTestFailureHelpMessage();
+ }
+ }
+ return (!failed);
+ }
+
+ public static void outputTestFailureHelpMessage() {
+ System.err.println(
+ "See ./ql/target/tmp/log/hive.log or ./itests/qtest/target/tmp/log/hive.log, or check " +
+ "./ql/target/surefire-reports or ./itests/qtest/target/surefire-reports/ for specific " +
+ "test cases logs.");
+ System.err.flush();
+ }
+
+ private static String[] cachedQvFileList = null;
+ private static ImmutableList<String> cachedDefaultQvFileList = null;
+ private static Pattern qvSuffix = Pattern.compile("_[0-9]+.qv$", Pattern.CASE_INSENSITIVE);
+
+ public static List<String> getVersionFiles(String queryDir, String tname) {
+ ensureQvFileList(queryDir);
+ List<String> result = getVersionFilesInternal(tname);
+ if (result == null) {
+ result = cachedDefaultQvFileList;
+ }
+ return result;
+ }
+
+ private static void ensureQvFileList(String queryDir) {
+ if (cachedQvFileList != null) {
+ return;
+ }
+ // Not thread-safe.
+ System.out.println("Getting versions from " + queryDir);
+ cachedQvFileList = (new File(queryDir)).list(new FilenameFilter() {
+ @Override
+ public boolean accept(File dir, String name) {
+ return name.toLowerCase().endsWith(".qv");
+ }
+ });
+ if (cachedQvFileList == null) {
+ return; // no files at all
+ }
+ Arrays.sort(cachedQvFileList, String.CASE_INSENSITIVE_ORDER);
+ List<String> defaults = getVersionFilesInternal("default");
+ cachedDefaultQvFileList = (defaults != null)
+ ? ImmutableList.copyOf(defaults) : ImmutableList.<String>of();
+ }
+
+ private static List<String> getVersionFilesInternal(String tname) {
+ if (cachedQvFileList == null) {
+ return new ArrayList<String>();
+ }
+ int pos = Arrays.binarySearch(cachedQvFileList, tname, String.CASE_INSENSITIVE_ORDER);
+ if (pos >= 0) {
+ throw new BuildException("Unexpected file list element: " + cachedQvFileList[pos]);
+ }
+ List<String> result = null;
+ for (pos = (-pos - 1); pos < cachedQvFileList.length; ++pos) {
+ String candidate = cachedQvFileList[pos];
+ if (candidate.length() <= tname.length()
+ || !tname.equalsIgnoreCase(candidate.substring(0, tname.length()))
+ || !qvSuffix.matcher(candidate.substring(tname.length())).matches()) {
+ break;
+ }
+ if (result == null) {
+ result = new ArrayList<String>();
+ }
+ result.add(candidate);
+ }
+ return result;
+ }
+
+ public void failed(int ecode, String fname, String debugHint) {
+ String command = SessionState.get() != null ? SessionState.get().getLastCommand() : null;
+ String message = "Client execution failed with error code = " + ecode +
+ (command != null ? " running \"" + command : "") + "\" fname=" + fname + " " +
+ (debugHint != null ? debugHint : "");
+ LOG.error(message);
+ Assert.fail(message);
+ }
+
+ // for negative tests, which is succeeded.. no need to print the query string
+ public void failed(String fname, String debugHint) {
+ Assert.fail(
+ "Client Execution was expected to fail, but succeeded with error code 0 for fname=" +
+ fname + (debugHint != null ? (" " + debugHint) : ""));
+ }
+
+ public void failedDiff(int ecode, String fname, String debugHint) {
+ String message =
+ "Client Execution succeeded but contained differences " +
+ "(error code = " + ecode + ") after executing " +
+ fname + (debugHint != null ? (" " + debugHint) : "");
+ LOG.error(message);
+ Assert.fail(message);
+ }
+
+ public void failed(Exception e, String fname, String debugHint) {
+ String command = SessionState.get() != null ? SessionState.get().getLastCommand() : null;
+ System.err.println("Failed query: " + fname);
+ System.err.flush();
+ Assert.fail("Unexpected exception " +
+ org.apache.hadoop.util.StringUtils.stringifyException(e) + "\n" +
+ (command != null ? " running " + command : "") +
+ (debugHint != null ? debugHint : ""));
+ }
+
+ public static void addTestsToSuiteFromQfileNames(
+ String qFileNamesFile,
+ Set<String> qFilesToExecute,
+ TestSuite suite,
+ Object setup,
+ SuiteAddTestFunctor suiteAddTestCallback) {
+ try {
+ File qFileNames = new File(qFileNamesFile);
+ FileReader fr = new FileReader(qFileNames.getCanonicalFile());
+ BufferedReader br = new BufferedReader(fr);
+ String fName = null;
+
+ while ((fName = br.readLine()) != null) {
+ if (fName.isEmpty() || fName.trim().equals("")) {
+ continue;
+ }
+
+ int eIdx = fName.indexOf('.');
+
+ if (eIdx == -1) {
+ continue;
+ }
+
+ String tName = fName.substring(0, eIdx);
+
+ if (qFilesToExecute.isEmpty() || qFilesToExecute.contains(fName)) {
+ suiteAddTestCallback.addTestToSuite(suite, setup, tName);
+ }
+ }
+ br.close();
+ } catch (Exception e) {
+ Assert.fail("Unexpected exception " + org.apache.hadoop.util.StringUtils.stringifyException(e));
+ }
+ }
+
+ public static void setupMetaStoreTableColumnStatsFor30TBTPCDSWorkload(HiveConf conf) {
+ Connection conn = null;
+ ArrayList<Statement> statements = new ArrayList<Statement>(); // list of Statements, PreparedStatements
+
+ try {
+ Properties props = new Properties(); // connection properties
+ props.put("user", conf.get("javax.jdo.option.ConnectionUserName"));
+ props.put("password", conf.get("javax.jdo.option.ConnectionPassword"));
+ conn = DriverManager.getConnection(conf.get("javax.jdo.option.ConnectionURL"), props);
+ ResultSet rs = null;
+ Statement s = conn.createStatement();
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Connected to metastore database ");
+ }
+
+ String mdbPath = HIVE_ROOT + "/data/files/tpcds-perf/metastore_export/";
+
+ // Setup the table column stats
+ BufferedReader br = new BufferedReader(
+ new FileReader(
+ new File(HIVE_ROOT + "/metastore/scripts/upgrade/derby/022-HIVE-11107.derby.sql")));
+ String command;
+
+ s.execute("DROP TABLE APP.TABLE_PARAMS");
+ s.execute("DROP TABLE APP.TAB_COL_STATS");
+ // Create the column stats table
+ while ((command = br.readLine()) != null) {
+ if (!command.endsWith(";")) {
+ continue;
+ }
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Going to run command : " + command);
+ }
+ try {
+ PreparedStatement psCommand = conn.prepareStatement(command.substring(0, command.length()-1));
+ statements.add(psCommand);
+ psCommand.execute();
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("successfully completed " + command);
+ }
+ } catch (SQLException e) {
+ LOG.info("Got SQL Exception " + e.getMessage());
+ }
+ }
+ br.close();
+
+ java.nio.file.Path tabColStatsCsv = FileSystems.getDefault().getPath(mdbPath, "csv" ,"TAB_COL_STATS.txt.bz2");
+ java.nio.file.Path tabParamsCsv = FileSystems.getDefault().getPath(mdbPath, "csv", "TABLE_PARAMS.txt.bz2");
+
+ // Set up the foreign key constraints properly in the TAB_COL_STATS data
+ String tmpBaseDir = System.getProperty(TEST_TMP_DIR_PROPERTY);
+ java.nio.file.Path tmpFileLoc1 = FileSystems.getDefault().getPath(tmpBaseDir, "TAB_COL_STATS.txt");
+ java.nio.file.Path tmpFileLoc2 = FileSystems.getDefault().getPath(tmpBaseDir, "TABLE_PARAMS.txt");
+
+ class MyComp implements Comparator<String> {
+ @Override
+ public int compare(String str1, String str2) {
+ if (str2.length() != str1.length()) {
+ return str2.length() - str1.length();
+ }
+ return str1.compareTo(str2);
+ }
+ }
+
+ final SortedMap<String, Integer> tableNameToID = new TreeMap<String, Integer>(new MyComp());
+
+ rs = s.executeQuery("SELECT * FROM APP.TBLS");
+ while(rs.next()) {
+ String tblName = rs.getString("TBL_NAME");
+ Integer tblId = rs.getInt("TBL_ID");
+ tableNameToID.put(tblName, tblId);
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Resultset : " + tblName + " | " + tblId);
+ }
+ }
+
+ final Map<String, Map<String, String>> data = new HashMap<>();
+ rs = s.executeQuery("select TBLS.TBL_NAME, a.COLUMN_NAME, a.TYPE_NAME from "
+ + "(select COLUMN_NAME, TYPE_NAME, SDS.SD_ID from APP.COLUMNS_V2 join APP.SDS on SDS.CD_ID = COLUMNS_V2.CD_ID) a"
+ + " join APP.TBLS on TBLS.SD_ID = a.SD_ID");
+ while (rs.next()) {
+ String tblName = rs.getString(1);
+ String colName = rs.getString(2);
+ String typeName = rs.getString(3);
+ Map<String, String> cols = data.get(tblName);
+ if (null == cols) {
+ cols = new HashMap<>();
+ }
+ cols.put(colName, typeName);
+ data.put(tblName, cols);
+ }
+
+ BufferedReader reader = new BufferedReader(new InputStreamReader(
+ new BZip2CompressorInputStream(Files.newInputStream(tabColStatsCsv, StandardOpenOption.READ))));
+
+ Stream<String> replaced = reader.lines().parallel().map(str-> {
+ String[] splits = str.split(",");
+ String tblName = splits[0];
+ String colName = splits[1];
+ Integer tblID = tableNameToID.get(tblName);
+ StringBuilder sb = new StringBuilder("default@"+tblName + "@" + colName + "@" + data.get(tblName).get(colName)+"@");
+ for (int i = 2; i < splits.length; i++) {
+ sb.append(splits[i]+"@");
+ }
+ // Add tbl_id and empty bitvector
+ return sb.append(tblID).append("@").toString();
+ });
+
+ Files.write(tmpFileLoc1, (Iterable<String>)replaced::iterator);
+ replaced.close();
+ reader.close();
+
+ BufferedReader reader2 = new BufferedReader(new InputStreamReader(
+ new BZip2CompressorInputStream(Files.newInputStream(tabParamsCsv, StandardOpenOption.READ))));
+ final Map<String,String> colStats = new ConcurrentHashMap<>();
+ Stream<String> replacedStream = reader2.lines().parallel().map(str-> {
+ String[] splits = str.split("_@");
+ String tblName = splits[0];
+ Integer tblId = tableNameToID.get(tblName);
+ Map<String,String> cols = data.get(tblName);
+ StringBuilder sb = new StringBuilder();
+ sb.append("{\"COLUMN_STATS\":{");
+ for (String colName : cols.keySet()) {
+ sb.append("\""+colName+"\":\"true\",");
+ }
+ sb.append("},\"BASIC_STATS\":\"true\"}");
+ colStats.put(tblId.toString(), sb.toString());
+
+ return tblId.toString() + "@" + splits[1];
+ });
+
+ Files.write(tmpFileLoc2, (Iterable<String>)replacedStream::iterator);
+ Files.write(tmpFileLoc2, (Iterable<String>)colStats.entrySet().stream()
+ .map(map->map.getKey()+"@COLUMN_STATS_ACCURATE@"+map.getValue())::iterator, StandardOpenOption.APPEND);
+
+ replacedStream.close();
+ reader2.close();
+ // Load the column stats and table params with 30 TB scale
+ String importStatement1 = "CALL SYSCS_UTIL.SYSCS_IMPORT_TABLE(null, '" + "TAB_COL_STATS" +
+ "', '" + tmpFileLoc1.toAbsolutePath().toString() +
+ "', '@', null, 'UTF-8', 1)";
+ String importStatement2 = "CALL SYSCS_UTIL.SYSCS_IMPORT_TABLE(null, '" + "TABLE_PARAMS" +
+ "', '" + tmpFileLoc2.toAbsolutePath().toString() +
+ "', '@', null, 'UTF-8', 1)";
+ try {
+ PreparedStatement psImport1 = conn.prepareStatement(importStatement1);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Going to execute : " + importStatement1);
+ }
+ statements.add(psImport1);
+ psImport1.execute();
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("successfully completed " + importStatement1);
+ }
+ PreparedStatement psImport2 = conn.prepareStatement(importStatement2);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Going to execute : " + importStatement2);
+ }
+ statements.add(psImport2);
+ psImport2.execute();
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("successfully completed " + importStatement2);
+ }
+ } catch (SQLException e) {
+ LOG.info("Got SQL Exception " + e.getMessage());
+ }
+ } catch (FileNotFoundException e1) {
+ LOG.info("Got File not found Exception " + e1.getMessage());
+ } catch (IOException e1) {
+ LOG.info("Got IOException " + e1.getMessage());
+ } catch (SQLException e1) {
+ LOG.info("Got SQLException " + e1.getMessage());
+ } finally {
+ // Statements and PreparedStatements
+ int i = 0;
+ while (!statements.isEmpty()) {
+ // PreparedStatement extend Statement
+ Statement st = statements.remove(i);
+ try {
+ if (st != null) {
+ st.close();
+ st = null;
+ }
+ } catch (SQLException sqle) {
+ }
+ }
+
+ //Connection
+ try {
+ if (conn != null) {
+ conn.close();
+ conn = null;
+ }
+ } catch (SQLException sqle) {
+ }
+ }
+ }
+
+ private static String getHiveRoot() {
+ String path;
+ if (System.getProperty("hive.root") != null) {
+ try {
+ path = new File(System.getProperty("hive.root")).getCanonicalPath();
+ } catch (IOException e) {
+ throw new RuntimeException("error getting hive.root", e);
+ }
+ } else {
+ path = new File("target").getAbsolutePath();
+ }
+ return ensurePathEndsInSlash(new File(path).getAbsolutePath());
+ }
+
+ public static String ensurePathEndsInSlash(String path) {
+ if (path == null) {
+ throw new NullPointerException("Path cannot be null");
+ }
+ if (path.endsWith(File.separator)) {
+ return path;
+ } else {
+ return path + File.separator;
+ }
+ }
+
+}