You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by jc...@apache.org on 2018/05/23 16:57:24 UTC
[1/4] hive git commit: Revert "HIVE-18117: Create TestCliDriver for
HDFS EC (Andrew Sherman, reviewed by Sahil Takiar)"
Repository: hive
Updated Branches:
refs/heads/master c8889f86d -> a1e6206cc
Revert "HIVE-18117: Create TestCliDriver for HDFS EC (Andrew Sherman, reviewed by Sahil Takiar)"
This reverts commit c8889f86d2f1e1f49064b05eff2c1b93504ba5bc.
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/29f57849
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/29f57849
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/29f57849
Branch: refs/heads/master
Commit: 29f5784941adf53bcecf0e899c348e547d779114
Parents: c8889f8
Author: Jesus Camacho Rodriguez <jc...@apache.org>
Authored: Wed May 23 09:51:31 2018 -0700
Committer: Jesus Camacho Rodriguez <jc...@apache.org>
Committed: Wed May 23 09:51:31 2018 -0700
----------------------------------------------------------------------
.../cli/TestErasureCodingHDFSCliDriver.java | 65 ----
.../test/resources/testconfiguration.properties | 6 -
.../hadoop/hive/cli/control/CliConfigs.java | 50 ---
.../org/apache/hadoop/hive/ql/QTestUtil.java | 60 +--
.../ql/processors/CommandProcessorFactory.java | 65 ++--
.../hive/ql/processors/ErasureProcessor.java | 383 -------------------
.../hadoop/hive/ql/processors/HiveCommand.java | 1 -
.../hadoop/hive/ql/session/SessionState.java | 5 -
.../processors/TestCommandProcessorFactory.java | 2 +-
.../queries/clientpositive/erasure_commands.q | 10 -
.../queries/clientpositive/erasure_simple.q | 51 ---
.../clientpositive/erasure_commands.q.out | 4 -
.../erasurecoding/erasure_commands.q.out | 8 -
.../erasurecoding/erasure_simple.q.out | 111 ------
.../apache/hadoop/hive/shims/Hadoop23Shims.java | 164 --------
.../apache/hadoop/hive/shims/HadoopShims.java | 107 ------
.../ptest2/conf/deployed/master-mr2.properties | 7 -
17 files changed, 40 insertions(+), 1059 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/29f57849/itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestErasureCodingHDFSCliDriver.java
----------------------------------------------------------------------
diff --git a/itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestErasureCodingHDFSCliDriver.java b/itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestErasureCodingHDFSCliDriver.java
deleted file mode 100644
index b459ffc..0000000
--- a/itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestErasureCodingHDFSCliDriver.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.cli;
-
-import java.io.File;
-import java.util.List;
-
-import org.apache.hadoop.hive.cli.control.CliAdapter;
-import org.apache.hadoop.hive.cli.control.CliConfigs;
-import org.junit.ClassRule;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.TestRule;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-import org.junit.runners.Parameterized.Parameters;
-
-/**
- * A Test Driver that can be used to run tests over hdfs directories that employ Erasure Coding.
- */
-@RunWith(Parameterized.class)
-public class TestErasureCodingHDFSCliDriver {
-
- static CliAdapter adapter = new CliConfigs.ErasureCodingHDFSCliConfig().getCliAdapter();
-
- @Parameters(name = "{0}")
- public static List<Object[]> getParameters() throws Exception {
- return adapter.getParameters();
- }
-
- // fields annotated with ClassRule must be public
- @ClassRule
- public static TestRule cliClassRule = adapter.buildClassRule();
-
- @Rule
- public TestRule cliTestRule = adapter.buildTestRule();
-
- private String name;
- private File qfile;
-
- public TestErasureCodingHDFSCliDriver(String name, File qfile) {
- this.name = name;
- this.qfile = qfile;
- }
-
- @Test
- public void testCliDriver() throws Exception {
- adapter.runTest(name, qfile);
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/29f57849/itests/src/test/resources/testconfiguration.properties
----------------------------------------------------------------------
diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties
index 81e9b66..6007d5a 100644
--- a/itests/src/test/resources/testconfiguration.properties
+++ b/itests/src/test/resources/testconfiguration.properties
@@ -1692,9 +1692,3 @@ druid.query.files=druidmini_test1.q,\
druidmini_floorTime.q
druid.kafka.query.files=druidkafkamini_basic.q
-
-# tests to be run by TestErasureCodingHDFSCliDriver and TestCliDriver
-erasurecoding.shared.query.files=erasure_commands.q
-
-# tests to be run only by TestErasureCodingHDFSCliDriver
-erasurecoding.only.query.files=erasure_simple.q
http://git-wip-us.apache.org/repos/asf/hive/blob/29f57849/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java
index fddd40f..7063adb 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java
@@ -57,7 +57,6 @@ public class CliConfigs {
excludesFrom(testConfigProps, "localSpark.only.query.files");
excludesFrom(testConfigProps, "druid.query.files");
excludesFrom(testConfigProps, "druid.kafka.query.files");
- excludesFrom(testConfigProps, "erasurecoding.only.query.files");
excludeQuery("fouter_join_ppr.q"); // Disabled in HIVE-19509
@@ -306,7 +305,6 @@ public class CliConfigs {
excludesFrom(testConfigProps, "minimr.query.files");
excludesFrom(testConfigProps, "minitez.query.files");
excludesFrom(testConfigProps, "encrypted.query.files");
- excludesFrom(testConfigProps, "erasurecoding.only.query.files");
setResultsDir("ql/src/test/results/clientpositive/perf/tez");
setLogDir("itests/qtest/target/qfile-results/clientpositive/tez");
@@ -662,52 +660,4 @@ public class CliConfigs {
}
}
}
-
- /**
- * Configuration for TestErasureCodingHDFSCliDriver.
- */
- public static class ErasureCodingHDFSCliConfig extends AbstractCliConfig {
- public ErasureCodingHDFSCliConfig() {
- super(CoreCliDriver.class);
- try {
- setQueryDir("ql/src/test/queries/clientpositive");
-
- includesFrom(testConfigProps, "erasurecoding.shared.query.files");
- includesFrom(testConfigProps, "erasurecoding.only.query.files");
-
- setResultsDir("ql/src/test/results/clientpositive/erasurecoding");
- setLogDir("itests/qtest/target/qfile-results/clientpositive");
-
- setInitScript("q_test_init_src.sql");
- setCleanupScript("q_test_cleanup_src.sql");
-
- setClusterType(MiniClusterType.mr);
- setFsType(QTestUtil.FsType.erasure_coded_hdfs);
- setHiveConfDir(getClusterType());
- } catch (Exception e) {
- throw new RuntimeException("can't construct cliconfig", e);
- }
- }
-
- /**
- * Set the appropriate conf dir based on the cluster type.
- */
- private void setHiveConfDir(MiniClusterType clusterType) {
- switch (clusterType) {
- case tez:
- setHiveConfDir("data/conf/tez");
- break;
- case spark:
- setHiveConfDir("data/conf/spark/standalone");
- break;
- case miniSparkOnYarn:
- setHiveConfDir("data/conf/spark/yarn-cluster");
- break;
- default:
- setHiveConfDir("data/conf");
- break;
- }
- }
- }
-
}
http://git-wip-us.apache.org/repos/asf/hive/blob/29f57849/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
index 2365fb7..57be0f0 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
@@ -109,7 +109,6 @@ import org.apache.hadoop.hive.ql.dataset.DatasetCollection;
import org.apache.hadoop.hive.ql.dataset.DatasetParser;
import org.apache.hadoop.hive.ql.dataset.Dataset;
import org.apache.hadoop.hive.shims.HadoopShims;
-import org.apache.hadoop.hive.shims.HadoopShims.HdfsErasureCodingShim;
import org.apache.hadoop.hive.shims.ShimLoader;
import org.apache.hive.common.util.StreamPrinter;
import org.apache.hive.druid.MiniDruidCluster;
@@ -144,7 +143,7 @@ public class QTestUtil {
static final Logger LOG = LoggerFactory.getLogger("QTestUtil");
private final static String defaultInitScript = "q_test_init.sql";
private final static String defaultCleanupScript = "q_test_cleanup.sql";
- private final String[] testOnlyCommands = new String[]{"crypto", "erasure"};
+ private final String[] testOnlyCommands = new String[]{"crypto"};
public static final String TEST_TMP_DIR_PROPERTY = "test.tmp.dir"; // typically target/tmp
private static final String BUILD_DIR_PROPERTY = "build.dir"; // typically target
@@ -152,11 +151,6 @@ public class QTestUtil {
public static final String TEST_SRC_TABLES_PROPERTY = "test.src.tables";
public static final String TEST_HIVE_USER_PROPERTY = "test.hive.user";
- /**
- * The Erasure Coding Policy to use in TestErasureCodingHDFSCliDriver.
- */
- private static final String DEFAULT_TEST_EC_POLICY = "RS-3-2-1024k";
-
private String testWarehouse;
private final String testFiles;
private final File datasetDir;
@@ -485,7 +479,6 @@ public class QTestUtil {
local,
hdfs,
encrypted_hdfs,
- erasure_coded_hdfs,
}
public enum MiniClusterType {
@@ -657,47 +650,25 @@ public class QTestUtil {
if (fsType == FsType.local) {
fs = FileSystem.getLocal(conf);
- } else if (fsType == FsType.hdfs || fsType == FsType.encrypted_hdfs|| fsType == FsType.erasure_coded_hdfs) {
+ } else if (fsType == FsType.hdfs || fsType == FsType.encrypted_hdfs) {
int numDataNodes = 4;
- // Setup before getting dfs
- switch (fsType) {
- case encrypted_hdfs:
+ if (fsType == FsType.encrypted_hdfs) {
// Set the security key provider so that the MiniDFS cluster is initialized
// with encryption
conf.set(SECURITY_KEY_PROVIDER_URI_NAME, getKeyProviderURI());
conf.setInt("fs.trash.interval", 50);
- break;
- case erasure_coded_hdfs:
- // We need more NameNodes for EC.
- // To fully exercise hdfs code paths we need 5 NameNodes for the RS-3-2-1024k policy.
- // With 6 NameNodes we can also run the RS-6-3-1024k policy.
- numDataNodes = 6;
- break;
- default:
- break;
- }
- dfs = shims.getMiniDfs(conf, numDataNodes, true, null);
- fs = dfs.getFileSystem();
+ dfs = shims.getMiniDfs(conf, numDataNodes, true, null);
+ fs = dfs.getFileSystem();
- // Setup after getting dfs
- switch (fsType) {
- case encrypted_hdfs:
// set up the java key provider for encrypted hdfs cluster
hes = shims.createHdfsEncryptionShim(fs, conf);
+
LOG.info("key provider is initialized");
- break;
- case erasure_coded_hdfs:
- // The Erasure policy can't be set in a q_test_init script as QTestUtil runs that code in
- // a mode that disallows test-only CommandProcessors.
- // Set the default policy on the root of the file system here.
- HdfsErasureCodingShim erasureCodingShim = shims.createHdfsErasureCodingShim(fs, conf);
- erasureCodingShim.enableErasureCodingPolicy(DEFAULT_TEST_EC_POLICY);
- erasureCodingShim.setErasureCodingPolicy(new Path("hdfs:///"), DEFAULT_TEST_EC_POLICY);
- break;
- default:
- break;
+ } else {
+ dfs = shims.getMiniDfs(conf, numDataNodes, true, null);
+ fs = dfs.getFileSystem();
}
} else {
throw new IllegalArgumentException("Unknown or unhandled fsType [" + fsType + "]");
@@ -1059,7 +1030,7 @@ public class QTestUtil {
LOG.warn("Trying to drop table " + e.getTableName() + ". But it does not exist.");
continue;
}
- db.dropTable(dbName, tblName, true, true, fsNeedsPurge(fsType));
+ db.dropTable(dbName, tblName, true, true, fsType == FsType.encrypted_hdfs);
}
}
if (!DEFAULT_DATABASE_NAME.equals(dbName)) {
@@ -2326,15 +2297,4 @@ public class QTestUtil {
public static void initEventNotificationPoll() throws Exception {
NotificationEventPoll.initialize(SessionState.get().getConf());
}
-
- /**
- * Should deleted test tables have their data purged.
- * @return true if data should be purged
- */
- private static boolean fsNeedsPurge(FsType type) {
- if (type == FsType.encrypted_hdfs || type == FsType.erasure_coded_hdfs) {
- return true;
- }
- return false;
- }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/29f57849/ql/src/java/org/apache/hadoop/hive/ql/processors/CommandProcessorFactory.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/processors/CommandProcessorFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/processors/CommandProcessorFactory.java
index f8b6a97..3d47991 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/processors/CommandProcessorFactory.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/processors/CommandProcessorFactory.java
@@ -20,7 +20,6 @@ package org.apache.hadoop.hive.ql.processors;
import static org.apache.commons.lang.StringUtils.isBlank;
-import java.io.IOException;
import java.sql.SQLException;
import java.util.HashSet;
import java.util.Set;
@@ -73,41 +72,35 @@ public final class CommandProcessorFactory {
return null;
}
switch (hiveCommand) {
- case SET:
- return new SetProcessor();
- case RESET:
- return new ResetProcessor();
- case DFS:
- SessionState ss = SessionState.get();
- return new DfsProcessor(ss.getConf());
- case ADD:
- return new AddResourceProcessor();
- case LIST:
- return new ListResourceProcessor();
- case LLAP_CLUSTER:
- return new LlapClusterResourceProcessor();
- case LLAP_CACHE:
- return new LlapCacheResourceProcessor();
- case DELETE:
- return new DeleteResourceProcessor();
- case COMPILE:
- return new CompileProcessor();
- case RELOAD:
- return new ReloadProcessor();
- case CRYPTO:
- try {
- return new CryptoProcessor(SessionState.get().getHdfsEncryptionShim(), conf);
- } catch (HiveException e) {
- throw new SQLException("Fail to start the command processor due to the exception: ", e);
- }
- case ERASURE:
- try {
- return new ErasureProcessor(conf);
- } catch (IOException e) {
- throw new SQLException("Fail to start the erasure command processor due to the exception: ", e);
- }
- default:
- throw new AssertionError("Unknown HiveCommand " + hiveCommand);
+ case SET:
+ return new SetProcessor();
+ case RESET:
+ return new ResetProcessor();
+ case DFS:
+ SessionState ss = SessionState.get();
+ return new DfsProcessor(ss.getConf());
+ case ADD:
+ return new AddResourceProcessor();
+ case LIST:
+ return new ListResourceProcessor();
+ case LLAP_CLUSTER:
+ return new LlapClusterResourceProcessor();
+ case LLAP_CACHE:
+ return new LlapCacheResourceProcessor();
+ case DELETE:
+ return new DeleteResourceProcessor();
+ case COMPILE:
+ return new CompileProcessor();
+ case RELOAD:
+ return new ReloadProcessor();
+ case CRYPTO:
+ try {
+ return new CryptoProcessor(SessionState.get().getHdfsEncryptionShim(), conf);
+ } catch (HiveException e) {
+ throw new SQLException("Fail to start the command processor due to the exception: ", e);
+ }
+ default:
+ throw new AssertionError("Unknown HiveCommand " + hiveCommand);
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/29f57849/ql/src/java/org/apache/hadoop/hive/ql/processors/ErasureProcessor.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/processors/ErasureProcessor.java b/ql/src/java/org/apache/hadoop/hive/ql/processors/ErasureProcessor.java
deleted file mode 100644
index 46114f5..0000000
--- a/ql/src/java/org/apache/hadoop/hive/ql/processors/ErasureProcessor.java
+++ /dev/null
@@ -1,383 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.ql.processors;
-
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.Comparator;
-import java.util.List;
-
-import com.google.common.base.Joiner;
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.CommandLineParser;
-import org.apache.commons.cli.GnuParser;
-import org.apache.commons.cli.Option;
-import org.apache.commons.cli.OptionBuilder;
-import org.apache.commons.cli.Options;
-import org.apache.commons.cli.ParseException;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.ql.session.SessionState;
-import org.apache.hadoop.hive.shims.HadoopShims;
-import org.apache.hadoop.hive.shims.HadoopShims.HdfsFileErasureCodingPolicy;
-import org.apache.hadoop.hive.shims.ShimLoader;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * This class processes hadoop commands used for Erasure Coding.
- * It is meant to be run only by Hive unit tests.
- *
- * The ‘Erasure’ commands implemented by this class allow test writers to use Erasure Coding in Hive.
- * Hdfs determines whether to use Erasure Coding for a file based on the presence of an Erasure
- * Coding Policy on the directory which contains the file.
- * These ‘Erasure’ commands can be used to manipulate Erasure Coding Policies.
- * These commands are similar to the user level commands provided by the ‘hdfs ec’ command as
- * documented at:
- * https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-hdfs/HDFSErasureCoding.html
- *
- * <ul>
- * <li>getPolicy: Get an erasure coding policy for a Path.
- * <li>enablePolicy: Enable an erasure coding policy.
- * <li>removePolicy: Remove an erasure coding policy.
- * <li>disablePolicy: Disable an erasure coding policy.
- * <li>setPolicy: Sets an erasure coding policy on a directory at the specified path
- * <li>unsetPolicy: Unsets an erasure coding policy on a directory at the specified path
- * <li>echo: Echo the parameters given to the command (not an ec command)
- * </ul>
- */
-public class ErasureProcessor implements CommandProcessor {
- private static final Logger LOG = LoggerFactory.getLogger(ErasureProcessor.class.getName());
-
- private HadoopShims.HdfsErasureCodingShim erasureCodingShim;
-
- ErasureProcessor(HiveConf config) throws IOException {
- HadoopShims hadoopShims = ShimLoader.getHadoopShims();
- FileSystem fileSystem = FileSystem.get(config);
- this.erasureCodingShim = hadoopShims.createHdfsErasureCodingShim(fileSystem, config);
- }
-
- private CommandLine parseCommandArgs(final Options opts, String[] args) throws ParseException {
- CommandLineParser parser = new GnuParser();
- return parser.parse(opts, args);
- }
-
- private CommandProcessorResponse returnErrorResponse(final String errmsg) {
- return new CommandProcessorResponse(1, "Erasure Processor Helper Failed: " + errmsg, null);
- }
-
- private void writeTestOutput(final String msg) {
- SessionState.get().out.println(msg);
- }
-
- @Override
- public CommandProcessorResponse run(String command) {
- String[] args = command.split("\\s+");
-
- if (args.length < 1) {
- return returnErrorResponse("Command arguments are empty.");
- }
-
- if (erasureCodingShim == null) {
- return returnErrorResponse("Hadoop erasure shim is not initialized.");
- }
-
- String action = args[0].toLowerCase();
- String[] params = Arrays.copyOfRange(args, 1, args.length);
-
- try {
- switch (action) {
- // note we switch on the lowercase command name
- case "disablepolicy":
- disablePolicy(params);
- break;
- case "echo":
- echo(params);
- break;
- case "enablepolicy":
- enablePolicy(params);
- break;
- case "getpolicy":
- getPolicy(params);
- break;
- case "listpolicies":
- listPolicies();
- break;
- case "setpolicy":
- setPolicy(params);
- break;
- case "removepolicy":
- removePolicy(params);
- break;
- case "unsetpolicy":
- unsetPolicy(params);
- break;
- default:
- return returnErrorResponse("Unknown erasure command action: " + action);
- }
- } catch (Exception e) {
- return returnErrorResponse(e.getMessage());
- }
-
- return new CommandProcessorResponse(0);
- }
-
- /**
- * Get an erasure coding policy for a Path.
- * @param params Parameters passed to the command.
- * @throws Exception if command failed.
- */
- private void getPolicy(String[] params) throws Exception {
- String command = "getPolicy";
- try {
- // getPolicy -path <path>
- Options getPolicyOptions = new Options();
-
- String pathOptionName = "path";
- Option policyOption = OptionBuilder.hasArg()
- .isRequired()
- .withLongOpt(pathOptionName)
- .withDescription("Path for which Policy should be fetched")
- .create();
- getPolicyOptions.addOption(policyOption);
-
- CommandLine args = parseCommandArgs(getPolicyOptions, params);
- String path = args.getOptionValue(pathOptionName);
-
- HdfsFileErasureCodingPolicy policy = erasureCodingShim.getErasureCodingPolicy(new Path(path));
- writeTestOutput("EC policy is '" + (policy != null ? policy.getName() : "REPLICATED") + "'");
-
- } catch (ParseException pe) {
- writeTestOutput("Error parsing options for " + command + " " + pe.getMessage());
- } catch (Exception e) {
- writeTestOutput("Caught exception running " + command + ": " + e.getMessage());
- throw new Exception("Cannot run " + command + ": " + e.getMessage(), e);
- }
- }
-
- /**
- * Echo the parameters given to the command.
- * @param params parameters which will be echoed
- */
- private void echo(String[] params) throws Exception {
- String command = "echo";
- try {
- writeTestOutput("ECHO " + Joiner.on(" ").join(params));
- } catch (Exception e) {
- writeTestOutput("Caught exception running " + command + ": " + e.getMessage());
- throw new Exception("Cannot run " + command + ": " + e.getMessage());
- }
- }
-
- /**
- * Enable an erasure coding policy.
- * @param params Parameters passed to the command.
- * @throws Exception If command failed.
- */
- private void enablePolicy(String[] params) throws Exception {
- String command = "enablePolicy";
- try {
- // enablePolicy -policy <policyName>
- Options enablePolicyOptions = new Options();
-
- String policyOptionName = "policy";
- Option policyOption = OptionBuilder.hasArg()
- .isRequired()
- .withLongOpt(policyOptionName)
- .withDescription("Policy to enable")
- .hasArg()
- .create();
- enablePolicyOptions.addOption(policyOption);
-
- CommandLine args = parseCommandArgs(enablePolicyOptions, params);
- String policyName = args.getOptionValue(policyOptionName);
-
- erasureCodingShim.enableErasureCodingPolicy(policyName);
- writeTestOutput("Enabled EC policy '" + policyName + "'");
- } catch (ParseException pe) {
- writeTestOutput("Error parsing options for " + command + " " + pe.getMessage());
- } catch (Exception e) {
- writeTestOutput("Caught exception running " + command + ": " + e.getMessage());
- throw new Exception("Cannot run " + command + ": " + e.getMessage());
- }
- }
-
- /**
- * Remove an erasure coding policy.
- * @param params Parameters passed to the command.
- * @throws Exception if command failed.
- */
- private void removePolicy(String[] params) throws Exception {
- String command = "removePolicy";
- try {
- // removePolicy -policy <policyName>
- Options removePolicyOptions = new Options();
-
- String policyOptionName = "policy";
- Option policyOption = OptionBuilder.hasArg()
- .isRequired()
- .withLongOpt(policyOptionName)
- .withDescription("Policy to remove")
- .create();
- removePolicyOptions.addOption(policyOption);
-
- CommandLine args = parseCommandArgs(removePolicyOptions, params);
- String policyName = args.getOptionValue(policyOptionName);
-
- erasureCodingShim.removeErasureCodingPolicy(policyName);
- writeTestOutput("Removed EC policy '" + policyName + "'");
- } catch (ParseException pe) {
- writeTestOutput("Error parsing options for " + command + " " + pe.getMessage());
- } catch (Exception e) {
- writeTestOutput("Caught exception running " + command + ": " + e.getMessage());
- throw new Exception("Cannot run " + command + ": " + e.getMessage());
- }
- }
-
- /**
- * Disable an erasure coding policy.
- * @param params Parameters passed to the command.
- * @throws Exception If command failed.
- */
- private void disablePolicy(String[] params) throws Exception {
- String command = "disablePolicy";
- try {
- // disablePolicy -policy <policyName>
- Options disablePolicyOptions = new Options();
-
- String policyOptionName = "policy";
- Option policyOption = OptionBuilder.hasArg()
- .isRequired()
- .withLongOpt(policyOptionName)
- .withDescription("Policy to disable")
- .create();
- disablePolicyOptions.addOption(policyOption);
-
- CommandLine args = parseCommandArgs(disablePolicyOptions, params);
- String policyName = args.getOptionValue(policyOptionName);
-
- erasureCodingShim.disableErasureCodingPolicy(policyName);
- writeTestOutput("Disabled EC policy '" + policyName + "'");
- } catch (ParseException pe) {
- writeTestOutput("Error parsing options for " + command + " " + pe.getMessage());
- } catch (Exception e) {
- writeTestOutput("Caught exception running " + command + ": " + e.getMessage());
- throw new Exception("Cannot run " + command + ": " + e.getMessage());
- }
- }
-
- /**
- * Sets an erasure coding policy on a directory at the specified path.
- * @param params Parameters passed to the command.
- * @throws Exception If command failed.
- */
- private void setPolicy(String[] params) throws Exception {
- String command = "setPolicy";
- try {
- // setPolicy -path <path> [-policy <policyName>]
- Options setPolicyOptions = new Options();
-
- String pathOptionName = "path";
- Option pathOption = OptionBuilder.hasArg()
- .isRequired()
- .withLongOpt(pathOptionName)
- .withDescription("Path to set policy on")
- .create();
- setPolicyOptions.addOption(pathOption);
-
- String policyOptionName = "policy";
- Option policyOption = OptionBuilder.hasArg()
- .withLongOpt(policyOptionName)
- .withDescription("Policy to set")
- .create();
- setPolicyOptions.addOption(policyOption);
-
- CommandLine args = parseCommandArgs(setPolicyOptions, params);
- String path = args.getOptionValue(pathOptionName);
- String policy = args.getOptionValue(policyOptionName);
-
- erasureCodingShim.setErasureCodingPolicy(new Path(path), policy);
- writeTestOutput("Set EC policy' " + policy);
- } catch (ParseException pe) {
- writeTestOutput("Error parsing options for " + command + " " + pe.getMessage());
- } catch (Exception e) {
- writeTestOutput("Caught exception running " + command + ": " + e.getMessage());
- throw new Exception("Cannot run " + command + ": " + e.getMessage());
- }
- }
-
- /**
- * Unsets an erasure coding policy on a directory at the specified path.
- * @param params Parameters passed to the command.
- * @throws Exception if command failed.
- */
- private void unsetPolicy(String[] params) throws Exception {
- String command = "unsetPolicy";
- try {
- // unsetPolicy -path <path>
- Options unsetPolicyOptions = new Options();
-
- String pathOptionName = "path";
- Option pathOption = OptionBuilder.hasArg()
- .isRequired()
- .withLongOpt(pathOptionName)
- .withDescription("Path to unset policy on")
- .create();
- unsetPolicyOptions.addOption(pathOption);
-
- CommandLine args = parseCommandArgs(unsetPolicyOptions, params);
- String path = args.getOptionValue(pathOptionName);
-
- erasureCodingShim.unsetErasureCodingPolicy(new Path(path));
- writeTestOutput("Unset EC policy");
- } catch (ParseException pe) {
- writeTestOutput("Error parsing options for " + command + " " + pe.getMessage());
- } catch (Exception e) {
- writeTestOutput("Caught exception running " + command + ": " + e.getMessage());
- throw new Exception("Cannot run " + command + ": " + e.getMessage());
- }
- }
-
- /**
- * Comparator the compares HdfsFileErasureCodingPolicy by name.
- */
- private Comparator<HdfsFileErasureCodingPolicy> nameComparator =
- Comparator.comparing(HdfsFileErasureCodingPolicy::getName);
-
- private void listPolicies() throws Exception {
- try {
- List<HdfsFileErasureCodingPolicy> erasureCodingPolicies =
- erasureCodingShim.getAllErasureCodingPolicies();
- erasureCodingPolicies.sort(nameComparator);
- if (erasureCodingPolicies.isEmpty()) {
- writeTestOutput("No EC Policies present");
- }
- for (HdfsFileErasureCodingPolicy policy : erasureCodingPolicies) {
- writeTestOutput("Policy: " + policy.getName() + " " + policy.getStatus());
- }
- } catch (Exception e) {
- throw new Exception("Cannot do language command: " + e.getMessage());
- }
- }
-
- @Override
- public void close() throws Exception {
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/29f57849/ql/src/java/org/apache/hadoop/hive/ql/processors/HiveCommand.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/processors/HiveCommand.java b/ql/src/java/org/apache/hadoop/hive/ql/processors/HiveCommand.java
index 74e3447..56c7516 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/processors/HiveCommand.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/processors/HiveCommand.java
@@ -30,7 +30,6 @@ public enum HiveCommand {
RESET(),
DFS(),
CRYPTO(true),
- ERASURE(true),
ADD(),
LIST(),
LLAP_CLUSTER(),
http://git-wip-us.apache.org/repos/asf/hive/blob/29f57849/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
index 6c56212..89129f9 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
@@ -241,11 +241,6 @@ public class SessionState {
*/
private Map<URI, HadoopShims.HdfsEncryptionShim> hdfsEncryptionShims = Maps.newHashMap();
- /**
- * Cache for Erasure Coding shims.
- */
- private Map<URI, HadoopShims.HdfsErasureCodingShim> erasureCodingShims;
-
private final String userName;
/**
http://git-wip-us.apache.org/repos/asf/hive/blob/29f57849/ql/src/test/org/apache/hadoop/hive/ql/processors/TestCommandProcessorFactory.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/processors/TestCommandProcessorFactory.java b/ql/src/test/org/apache/hadoop/hive/ql/processors/TestCommandProcessorFactory.java
index 0c960f2..de43c28 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/processors/TestCommandProcessorFactory.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/processors/TestCommandProcessorFactory.java
@@ -28,7 +28,7 @@ import org.junit.Test;
public class TestCommandProcessorFactory {
- private final String[] testOnlyCommands = new String[]{"crypto", "erasure"};
+ private final String[] testOnlyCommands = new String[]{"crypto"};
private HiveConf conf;
http://git-wip-us.apache.org/repos/asf/hive/blob/29f57849/ql/src/test/queries/clientpositive/erasure_commands.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/erasure_commands.q b/ql/src/test/queries/clientpositive/erasure_commands.q
deleted file mode 100644
index c8e09b8..0000000
--- a/ql/src/test/queries/clientpositive/erasure_commands.q
+++ /dev/null
@@ -1,10 +0,0 @@
---! qt:dataset:src
--- simple test to show ERASURE commands running on local fs (TestCliDriver) or hdfs (TestErasureCodingHDFSCliDriver).
-
-ERASURE echo listOfPolicies output is:;
-ERASURE listPolicies;
-
--- what is the policy on the root of the fs?
-ERASURE echo original policy on /;
-ERASURE getPolicy --path /;
-
http://git-wip-us.apache.org/repos/asf/hive/blob/29f57849/ql/src/test/queries/clientpositive/erasure_simple.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/erasure_simple.q b/ql/src/test/queries/clientpositive/erasure_simple.q
deleted file mode 100644
index c08409c..0000000
--- a/ql/src/test/queries/clientpositive/erasure_simple.q
+++ /dev/null
@@ -1,51 +0,0 @@
---! qt:dataset:src
-
--- Test Erasure Coding Policies
-
-ERASURE echo listPolicies originally was;
-ERASURE listPolicies;
-ERASURE enablePolicy --policy RS-10-4-1024k;
-ERASURE echo listPolicies after enablePolicy;
-ERASURE listPolicies;
-
-dfs ${system:test.dfs.mkdir} hdfs:///tmp/erasure_coding1;
-
-ERASURE echo original policy on erasure_coding1;
-ERASURE getPolicy --path hdfs:///tmp/erasure_coding1;
-
-ERASURE echo set the default policy on erasure_coding1;
-ERASURE setPolicy --path hdfs:///tmp/erasure_coding1 --policy RS-10-4-1024k;
-
-ERASURE echo new policy on erasure_coding1;
-ERASURE getPolicy --path hdfs:///tmp/erasure_coding1;
-
-ERASURE echo unset the default policy on erasure_coding1;
-ERASURE unsetPolicy --path hdfs:///tmp/erasure_coding1;
-ERASURE getPolicy --path hdfs:///tmp/erasure_coding1;
-
-create table erasure_table (a int) location 'hdfs:///tmp/erasure_coding1/location1';
-
-insert into erasure_table values(4);
-
-select * from erasure_table;
-
-drop table if exists erasure_table2;
-create table erasure_table2 like src location 'hdfs:///tmp/erasure_coding1/location2';
-insert overwrite table erasure_table2
-select key, value from src;
-
-ERASURE echo show table extended like erasure_table2;
-show table extended like erasure_table2;
-
-ERASURE echo SHOW TBLPROPERTIES erasure_table2;
-SHOW TBLPROPERTIES erasure_table2;
-
-ERASURE echo unset the default policy on erasure_coding1;
-ERASURE unsetPolicy --path hdfs:///tmp/erasure_coding1;
-
-dfs -rmr hdfs:///tmp/erasure_coding1;
-
-ERASURE echo disablePolicy RS-10-4-1024k;
-ERASURE disablePolicy --policy RS-10-4-1024k;
-
-
http://git-wip-us.apache.org/repos/asf/hive/blob/29f57849/ql/src/test/results/clientpositive/erasure_commands.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/erasure_commands.q.out b/ql/src/test/results/clientpositive/erasure_commands.q.out
deleted file mode 100644
index 92d2f6a..0000000
--- a/ql/src/test/results/clientpositive/erasure_commands.q.out
+++ /dev/null
@@ -1,4 +0,0 @@
-ECHO listOfPolicies output is:
-No EC Policies present
-ECHO original policy on /
-EC policy is 'REPLICATED'
http://git-wip-us.apache.org/repos/asf/hive/blob/29f57849/ql/src/test/results/clientpositive/erasurecoding/erasure_commands.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/erasurecoding/erasure_commands.q.out b/ql/src/test/results/clientpositive/erasurecoding/erasure_commands.q.out
deleted file mode 100644
index e927e05..0000000
--- a/ql/src/test/results/clientpositive/erasurecoding/erasure_commands.q.out
+++ /dev/null
@@ -1,8 +0,0 @@
-ECHO listOfPolicies output is:
-Policy: RS-10-4-1024k DISABLED
-Policy: RS-3-2-1024k ENABLED
-Policy: RS-6-3-1024k ENABLED
-Policy: RS-LEGACY-6-3-1024k DISABLED
-Policy: XOR-2-1-1024k DISABLED
-ECHO original policy on /
-EC policy is 'RS-3-2-1024k'
http://git-wip-us.apache.org/repos/asf/hive/blob/29f57849/ql/src/test/results/clientpositive/erasurecoding/erasure_simple.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/erasurecoding/erasure_simple.q.out b/ql/src/test/results/clientpositive/erasurecoding/erasure_simple.q.out
deleted file mode 100644
index 01f6015..0000000
--- a/ql/src/test/results/clientpositive/erasurecoding/erasure_simple.q.out
+++ /dev/null
@@ -1,111 +0,0 @@
-ECHO listPolicies originally was
-Policy: RS-10-4-1024k DISABLED
-Policy: RS-3-2-1024k ENABLED
-Policy: RS-6-3-1024k ENABLED
-Policy: RS-LEGACY-6-3-1024k DISABLED
-Policy: XOR-2-1-1024k DISABLED
-Enabled EC policy 'RS-10-4-1024k'
-ECHO listPolicies after enablePolicy
-Policy: RS-10-4-1024k ENABLED
-Policy: RS-3-2-1024k ENABLED
-Policy: RS-6-3-1024k ENABLED
-Policy: RS-LEGACY-6-3-1024k DISABLED
-Policy: XOR-2-1-1024k DISABLED
-ECHO original policy on erasure_coding1
-EC policy is 'RS-3-2-1024k'
-ECHO set the default policy on erasure_coding1
-Set EC policy' RS-10-4-1024k
-ECHO new policy on erasure_coding1
-EC policy is 'RS-10-4-1024k'
-ECHO unset the default policy on erasure_coding1
-Unset EC policy
-EC policy is 'RS-3-2-1024k'
-PREHOOK: query: create table erasure_table (a int) location 'hdfs://### HDFS PATH ###'
-PREHOOK: type: CREATETABLE
-PREHOOK: Input: hdfs://### HDFS PATH ###
-PREHOOK: Output: database:default
-PREHOOK: Output: default@erasure_table
-POSTHOOK: query: create table erasure_table (a int) location 'hdfs://### HDFS PATH ###'
-POSTHOOK: type: CREATETABLE
-POSTHOOK: Input: hdfs://### HDFS PATH ###
-POSTHOOK: Output: database:default
-POSTHOOK: Output: default@erasure_table
-PREHOOK: query: insert into erasure_table values(4)
-PREHOOK: type: QUERY
-PREHOOK: Input: _dummy_database@_dummy_table
-PREHOOK: Output: default@erasure_table
-POSTHOOK: query: insert into erasure_table values(4)
-POSTHOOK: type: QUERY
-POSTHOOK: Input: _dummy_database@_dummy_table
-POSTHOOK: Output: default@erasure_table
-POSTHOOK: Lineage: erasure_table.a SCRIPT []
-PREHOOK: query: select * from erasure_table
-PREHOOK: type: QUERY
-PREHOOK: Input: default@erasure_table
-PREHOOK: Output: hdfs://### HDFS PATH ###
-POSTHOOK: query: select * from erasure_table
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@erasure_table
-POSTHOOK: Output: hdfs://### HDFS PATH ###
-4
-PREHOOK: query: drop table if exists erasure_table2
-PREHOOK: type: DROPTABLE
-POSTHOOK: query: drop table if exists erasure_table2
-POSTHOOK: type: DROPTABLE
-PREHOOK: query: create table erasure_table2 like src location 'hdfs://### HDFS PATH ###'
-PREHOOK: type: CREATETABLE
-PREHOOK: Input: hdfs://### HDFS PATH ###
-PREHOOK: Output: database:default
-PREHOOK: Output: default@erasure_table2
-POSTHOOK: query: create table erasure_table2 like src location 'hdfs://### HDFS PATH ###'
-POSTHOOK: type: CREATETABLE
-POSTHOOK: Input: hdfs://### HDFS PATH ###
-POSTHOOK: Output: database:default
-POSTHOOK: Output: default@erasure_table2
-PREHOOK: query: insert overwrite table erasure_table2
-select key, value from src
-PREHOOK: type: QUERY
-PREHOOK: Input: default@src
-PREHOOK: Output: default@erasure_table2
-POSTHOOK: query: insert overwrite table erasure_table2
-select key, value from src
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@src
-POSTHOOK: Output: default@erasure_table2
-POSTHOOK: Lineage: erasure_table2.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
-POSTHOOK: Lineage: erasure_table2.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
-ECHO show table extended like erasure_table2
-PREHOOK: query: show table extended like erasure_table2
-PREHOOK: type: SHOW_TABLESTATUS
-POSTHOOK: query: show table extended like erasure_table2
-POSTHOOK: type: SHOW_TABLESTATUS
-tableName:erasure_table2
-#### A masked pattern was here ####
-location:hdfs://### HDFS PATH ###
-inputformat:org.apache.hadoop.mapred.TextInputFormat
-outputformat:org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
-columns:struct columns { string key, string value}
-partitioned:false
-partitionColumns:
-totalNumberFiles:1
-totalFileSize:5812
-maxFileSize:5812
-minFileSize:5812
-#### A masked pattern was here ####
-
-ECHO SHOW TBLPROPERTIES erasure_table2
-PREHOOK: query: SHOW TBLPROPERTIES erasure_table2
-PREHOOK: type: SHOW_TBLPROPERTIES
-POSTHOOK: query: SHOW TBLPROPERTIES erasure_table2
-POSTHOOK: type: SHOW_TBLPROPERTIES
-COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}}
-numFiles 1
-numRows 500
-rawDataSize 5312
-totalSize 5812
-#### A masked pattern was here ####
-ECHO unset the default policy on erasure_coding1
-Unset EC policy
-#### A masked pattern was here ####
-ECHO disablePolicy RS-10-4-1024k
-Disabled EC policy 'RS-10-4-1024k'
http://git-wip-us.apache.org/repos/asf/hive/blob/29f57849/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
----------------------------------------------------------------------
diff --git a/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java b/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
index 5a77122..ec06a88 100644
--- a/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
+++ b/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
@@ -65,8 +65,6 @@ import org.apache.hadoop.hdfs.MiniDFSNNTopology;
import org.apache.hadoop.hdfs.client.HdfsAdmin;
import org.apache.hadoop.hdfs.protocol.DirectoryListing;
import org.apache.hadoop.hdfs.protocol.EncryptionZone;
-import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
-import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyInfo;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.HdfsLocatedFileStatus;
import org.apache.hadoop.io.LongWritable;
@@ -1443,166 +1441,4 @@ public class Hadoop23Shims extends HadoopShimsSecure {
return set;
}
- private static Boolean hdfsErasureCodingSupport;
-
- /**
- * @return true if the runtime version of hdfs supports erasure coding
- */
- private static synchronized boolean isHdfsErasureCodingSupported() {
- if (hdfsErasureCodingSupport == null) {
- Method m = null;
-
- try {
- m = HdfsAdmin.class.getMethod("getErasureCodingPolicies");
- } catch (NoSuchMethodException e) {
- // This version of Hadoop does not support HdfsAdmin.getErasureCodingPolicies().
- // Hadoop 3.0.0 introduces this new method.
- }
- hdfsErasureCodingSupport = (m != null);
- }
-
- return hdfsErasureCodingSupport;
- }
-
- /**
- * Returns a new instance of the HdfsErasureCoding shim.
- *
- * @param fs a FileSystem object
- * @param conf a Configuration object
- * @return a new instance of the HdfsErasureCoding shim.
- * @throws IOException If an error occurred while creating the instance.
- */
- @Override
- public HadoopShims.HdfsErasureCodingShim createHdfsErasureCodingShim(FileSystem fs,
- Configuration conf) throws IOException {
- if (isHdfsErasureCodingSupported()) {
- URI uri = fs.getUri();
- if ("hdfs".equals(uri.getScheme())) {
- return new HdfsErasureCodingShim(uri, conf);
- }
- }
- return new HadoopShims.NoopHdfsErasureCodingShim();
- }
-
- /**
- * Information about an Erasure Coding Policy.
- */
- private static class HdfsFileErasureCodingPolicyImpl implements HdfsFileErasureCodingPolicy {
- private final String name;
- private final String status;
-
- HdfsFileErasureCodingPolicyImpl(String name, String status) {
- this.name = name;
- this.status = status;
- }
-
- HdfsFileErasureCodingPolicyImpl(String name) {
- this(name, null);
- }
-
- @Override
- public String getName() {
- return name;
- }
-
- @Override
- public String getStatus() {
- return status;
- }
- }
-
- /**
- * This class encapsulates methods used to get Erasure Coding information from
- * HDFS paths in order to to provide commands similar to those provided by the hdfs ec command.
- * https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-hdfs/HDFSErasureCoding.html
- */
- public static class HdfsErasureCodingShim implements HadoopShims.HdfsErasureCodingShim {
- /**
- * Gets information about HDFS encryption zones.
- */
- private HdfsAdmin hdfsAdmin = null;
-
- private final Configuration conf;
-
- HdfsErasureCodingShim(URI uri, Configuration conf) throws IOException {
- this.conf = conf;
- this.hdfsAdmin = new HdfsAdmin(uri, conf);
- }
-
- /**
- * Lists all (enabled, disabled and removed) erasure coding policies registered in HDFS.
- * @return a list of erasure coding policies
- */
- @Override
- public List<HdfsFileErasureCodingPolicy> getAllErasureCodingPolicies() throws IOException {
- ErasureCodingPolicyInfo[] erasureCodingPolicies = hdfsAdmin.getErasureCodingPolicies();
- List<HdfsFileErasureCodingPolicy> policies = new ArrayList<>(erasureCodingPolicies.length);
- for (ErasureCodingPolicyInfo erasureCodingPolicy : erasureCodingPolicies) {
- policies.add(new HdfsFileErasureCodingPolicyImpl(erasureCodingPolicy.getPolicy().getName(),
- erasureCodingPolicy.getState().toString()));
- }
- return policies;
- }
-
-
- /**
- * Enable an erasure coding policy.
- * @param ecPolicyName the name of the erasure coding policy
- */
- @Override
- public void enableErasureCodingPolicy(String ecPolicyName) throws IOException {
- hdfsAdmin.enableErasureCodingPolicy(ecPolicyName);
- }
-
- /**
- * Sets an erasure coding policy on a directory at the specified path.
- * @param path a directory in HDFS
- * @param ecPolicyName the name of the erasure coding policy
- */
- @Override
- public void setErasureCodingPolicy(Path path, String ecPolicyName) throws IOException {
- hdfsAdmin.setErasureCodingPolicy(path, ecPolicyName);
- }
-
- /**
- * Get details of the erasure coding policy of a file or directory at the specified path.
- * @param path an hdfs file or directory
- * @return an erasure coding policy
- */
- @Override
- public HdfsFileErasureCodingPolicy getErasureCodingPolicy(Path path) throws IOException {
- ErasureCodingPolicy erasureCodingPolicy = hdfsAdmin.getErasureCodingPolicy(path);
- if (erasureCodingPolicy == null) {
- return null;
- }
- return new HdfsFileErasureCodingPolicyImpl(erasureCodingPolicy.getName());
- }
-
- /**
- * Unset an erasure coding policy set by a previous call to setPolicy on a directory.
- * @param path a directory in HDFS
- */
- @Override
- public void unsetErasureCodingPolicy(Path path) throws IOException {
- hdfsAdmin.unsetErasureCodingPolicy(path);
- }
-
- /**
- * Remove an erasure coding policy.
- * @param ecPolicyName the name of the erasure coding policy
- */
- @Override
- public void removeErasureCodingPolicy(String ecPolicyName) throws IOException {
- hdfsAdmin.removeErasureCodingPolicy(ecPolicyName);
- }
-
- /**
- * Disable an erasure coding policy.
- * @param ecPolicyName the name of the erasure coding policy
- */
- @Override
- public void disableErasureCodingPolicy(String ecPolicyName) throws IOException {
- hdfsAdmin.disableErasureCodingPolicy(ecPolicyName);
- }
- }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/29f57849/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java
----------------------------------------------------------------------
diff --git a/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java b/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java
index 2e84ca9..b890817 100644
--- a/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java
+++ b/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java
@@ -24,7 +24,6 @@ import java.net.URI;
import java.nio.ByteBuffer;
import java.security.AccessControlException;
import java.security.NoSuchAlgorithmException;
-import java.util.Collections;
import java.util.Comparator;
import java.util.List;
import java.util.Set;
@@ -634,112 +633,6 @@ public interface HadoopShims {
*/
public HdfsEncryptionShim createHdfsEncryptionShim(FileSystem fs, Configuration conf) throws IOException;
- /**
- * Information about an Erasure Coding Policy.
- */
- interface HdfsFileErasureCodingPolicy {
- String getName();
- String getStatus();
- }
-
- /**
- * This interface encapsulates methods used to get Erasure Coding information from
- * HDFS paths in order to to provide commands similar to those provided by the hdfs ec command.
- * https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-hdfs/HDFSErasureCoding.html
- */
- interface HdfsErasureCodingShim {
- /**
- * Lists all (enabled, disabled and removed) erasure coding policies registered in HDFS.
- * @return a list of erasure coding policies
- */
- List<HdfsFileErasureCodingPolicy> getAllErasureCodingPolicies() throws IOException;
-
- /**
- * Enable an erasure coding policy.
- * @param ecPolicyName the name of the erasure coding policy
- */
- void enableErasureCodingPolicy(String ecPolicyName) throws IOException;
-
- /**
- * Sets an erasure coding policy on a directory at the specified path.
- * @param path a directory in HDFS
- * @param ecPolicyName the name of the erasure coding policy
- */
- void setErasureCodingPolicy(Path path, String ecPolicyName) throws IOException;
-
- /**
- * Get details of the erasure coding policy of a file or directory at the specified path.
- * @param path an hdfs file or directory
- * @return an erasure coding policy
- */
- HdfsFileErasureCodingPolicy getErasureCodingPolicy(Path path) throws IOException;
-
- /**
- * Unset an erasure coding policy set by a previous call to setPolicy on a directory.
- * @param path a directory in HDFS
- */
- void unsetErasureCodingPolicy(Path path) throws IOException;
-
- /**
- * Remove an erasure coding policy.
- * @param ecPolicyName the name of the erasure coding policy
- */
- void removeErasureCodingPolicy(String ecPolicyName) throws IOException;
-
- /**
- * Disable an erasure coding policy.
- * @param ecPolicyName the name of the erasure coding policy
- */
- void disableErasureCodingPolicy(String ecPolicyName) throws IOException;
- }
-
- /**
- * This is a dummy class used when the hadoop version does not support hdfs Erasure Coding.
- */
- class NoopHdfsErasureCodingShim implements HadoopShims.HdfsErasureCodingShim {
-
- @Override
- public List<HadoopShims.HdfsFileErasureCodingPolicy> getAllErasureCodingPolicies() {
- return Collections.emptyList();
- }
-
- @Override
- public void enableErasureCodingPolicy(String ecPolicyName) throws IOException {
- }
-
- @Override
- public void setErasureCodingPolicy(Path path, String ecPolicyName) throws IOException {
- }
-
- @Override
- public HdfsFileErasureCodingPolicy getErasureCodingPolicy(Path path) throws IOException {
- return null;
- }
-
- @Override
- public void unsetErasureCodingPolicy(Path path) throws IOException {
- }
-
- @Override
- public void removeErasureCodingPolicy(String ecPolicyName) throws IOException {
- }
-
- @Override
- public void disableErasureCodingPolicy(String ecPolicyName) throws IOException {
- }
-
- }
-
- /**
- * Returns a new instance of the HdfsErasureCoding shim.
- *
- * @param fs a FileSystem object
- * @param conf a Configuration object
- * @return a new instance of the HdfsErasureCoding shim.
- * @throws IOException If an error occurred while creating the instance.
- */
- HadoopShims.HdfsErasureCodingShim createHdfsErasureCodingShim(FileSystem fs, Configuration conf) throws IOException;
-
public Path getPathWithoutSchemeAndAuthority(Path path);
/**
http://git-wip-us.apache.org/repos/asf/hive/blob/29f57849/testutils/ptest2/conf/deployed/master-mr2.properties
----------------------------------------------------------------------
diff --git a/testutils/ptest2/conf/deployed/master-mr2.properties b/testutils/ptest2/conf/deployed/master-mr2.properties
index f04c0ce..7edc307 100644
--- a/testutils/ptest2/conf/deployed/master-mr2.properties
+++ b/testutils/ptest2/conf/deployed/master-mr2.properties
@@ -172,10 +172,3 @@ qFileTest.miniLlapLocal.batchSize = 30
qFileTest.miniLlapLocal.queryFilesProperty = qfile
qFileTest.miniLlapLocal.include = normal
qFileTest.miniLlapLocal.groups.normal = mainProperties.${minillaplocal.query.files} mainProperties.${minillaplocal.shared.query.files}
-
-qFileTest.erasurecodingCli.driver = TestErasureCodingHDFSCliDriver
-qFileTest.erasurecodingCli.directory = ql/src/test/queries/clientpositive
-qFileTest.erasurecodingCli.batchSize = 15
-qFileTest.erasurecodingCli.queryFilesProperty = qfile
-qFileTest.erasurecodingCli.include = normal
-qFileTest.erasurecodingCli.groups.normal = mainProperties.${erasurecoding.only.query.files} mainProperties.${erasurecoding.shared.query.files}
[3/4] hive git commit: Revert "HIVE-19500: Prevent multiple
selectivity estimations for the same variable in conjuctions (Zoltan
Haindrich reviewed by Ashutosh Chauhan)"
Posted by jc...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/a1e6206c/ql/src/test/results/clientpositive/stat_estimate_related_col.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/stat_estimate_related_col.q.out b/ql/src/test/results/clientpositive/stat_estimate_related_col.q.out
deleted file mode 100644
index 8cc0fbc..0000000
--- a/ql/src/test/results/clientpositive/stat_estimate_related_col.q.out
+++ /dev/null
@@ -1,784 +0,0 @@
-PREHOOK: query: drop table if exists t1
-PREHOOK: type: DROPTABLE
-POSTHOOK: query: drop table if exists t1
-POSTHOOK: type: DROPTABLE
-PREHOOK: query: drop table if exists t8
-PREHOOK: type: DROPTABLE
-POSTHOOK: query: drop table if exists t8
-POSTHOOK: type: DROPTABLE
-PREHOOK: query: create table t1 (a integer,b integer)
-PREHOOK: type: CREATETABLE
-PREHOOK: Output: database:default
-PREHOOK: Output: default@t1
-POSTHOOK: query: create table t1 (a integer,b integer)
-POSTHOOK: type: CREATETABLE
-POSTHOOK: Output: database:default
-POSTHOOK: Output: default@t1
-PREHOOK: query: create table t8 like t1
-PREHOOK: type: CREATETABLE
-PREHOOK: Output: database:default
-PREHOOK: Output: default@t8
-POSTHOOK: query: create table t8 like t1
-POSTHOOK: type: CREATETABLE
-POSTHOOK: Output: database:default
-POSTHOOK: Output: default@t8
-PREHOOK: query: insert into t1 values (1,1),(2,2),(3,3),(4,4),(5,5)
-PREHOOK: type: QUERY
-PREHOOK: Input: _dummy_database@_dummy_table
-PREHOOK: Output: default@t1
-POSTHOOK: query: insert into t1 values (1,1),(2,2),(3,3),(4,4),(5,5)
-POSTHOOK: type: QUERY
-POSTHOOK: Input: _dummy_database@_dummy_table
-POSTHOOK: Output: default@t1
-POSTHOOK: Lineage: t1.a SCRIPT []
-POSTHOOK: Lineage: t1.b SCRIPT []
-PREHOOK: query: insert into t8
-select * from t1 union all select * from t1 union all select * from t1 union all select * from t1 union all
-select * from t1 union all select * from t1 union all select * from t1 union all select * from t1
-PREHOOK: type: QUERY
-PREHOOK: Input: default@t1
-PREHOOK: Output: default@t8
-POSTHOOK: query: insert into t8
-select * from t1 union all select * from t1 union all select * from t1 union all select * from t1 union all
-select * from t1 union all select * from t1 union all select * from t1 union all select * from t1
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@t1
-POSTHOOK: Output: default@t8
-POSTHOOK: Lineage: t8.a EXPRESSION [(t1)t1.FieldSchema(name:a, type:int, comment:null), ]
-POSTHOOK: Lineage: t8.b EXPRESSION [(t1)t1.FieldSchema(name:b, type:int, comment:null), ]
-PREHOOK: query: analyze table t1 compute statistics for columns
-PREHOOK: type: ANALYZE_TABLE
-PREHOOK: Input: default@t1
-PREHOOK: Output: default@t1
-#### A masked pattern was here ####
-POSTHOOK: query: analyze table t1 compute statistics for columns
-POSTHOOK: type: ANALYZE_TABLE
-POSTHOOK: Input: default@t1
-POSTHOOK: Output: default@t1
-#### A masked pattern was here ####
-PREHOOK: query: analyze table t8 compute statistics for columns
-PREHOOK: type: ANALYZE_TABLE
-PREHOOK: Input: default@t8
-PREHOOK: Output: default@t8
-#### A masked pattern was here ####
-POSTHOOK: query: analyze table t8 compute statistics for columns
-POSTHOOK: type: ANALYZE_TABLE
-POSTHOOK: Input: default@t8
-POSTHOOK: Output: default@t8
-#### A masked pattern was here ####
-PREHOOK: query: select sum(a) from t8 where b in (2,3) group by b
-PREHOOK: type: QUERY
-PREHOOK: Input: default@t8
-#### A masked pattern was here ####
-POSTHOOK: query: select sum(a) from t8 where b in (2,3) group by b
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@t8
-#### A masked pattern was here ####
-PREHOOK: query: explain analyze select sum(a) from t8 where b in (2,3) group by b
-PREHOOK: type: QUERY
-POSTHOOK: query: explain analyze select sum(a) from t8 where b in (2,3) group by b
-POSTHOOK: type: QUERY
-STAGE DEPENDENCIES:
- Stage-1 is a root stage
- Stage-0 depends on stages: Stage-1
-
-STAGE PLANS:
- Stage: Stage-1
- Map Reduce
- Map Operator Tree:
- TableScan
- alias: t8
- Statistics: Num rows: 40/40 Data size: 320 Basic stats: COMPLETE Column stats: COMPLETE
- Filter Operator
- predicate: (b) IN (2, 3) (type: boolean)
- Statistics: Num rows: 16/16 Data size: 128 Basic stats: COMPLETE Column stats: COMPLETE
- Group By Operator
- aggregations: sum(a)
- keys: b (type: int)
- mode: hash
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 2/2 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE
- Reduce Output Operator
- key expressions: _col0 (type: int)
- sort order: +
- Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 2/2 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE
- value expressions: _col1 (type: bigint)
- Execution mode: vectorized
- Reduce Operator Tree:
- Group By Operator
- aggregations: sum(VALUE._col0)
- keys: KEY._col0 (type: int)
- mode: mergepartial
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 2/2 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE
- Select Operator
- expressions: _col1 (type: bigint)
- outputColumnNames: _col0
- Statistics: Num rows: 2/2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE
- File Output Operator
- compressed: false
- Statistics: Num rows: 2/2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-
- Stage: Stage-0
- Fetch Operator
- limit: -1
- Processor Tree:
- ListSink
-
-PREHOOK: query: select sum(a) from t8 where b=2 group by b
-PREHOOK: type: QUERY
-PREHOOK: Input: default@t8
-#### A masked pattern was here ####
-POSTHOOK: query: select sum(a) from t8 where b=2 group by b
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@t8
-#### A masked pattern was here ####
-PREHOOK: query: explain analyze select sum(a) from t8 where b=2 group by b
-PREHOOK: type: QUERY
-POSTHOOK: query: explain analyze select sum(a) from t8 where b=2 group by b
-POSTHOOK: type: QUERY
-STAGE DEPENDENCIES:
- Stage-1 is a root stage
- Stage-0 depends on stages: Stage-1
-
-STAGE PLANS:
- Stage: Stage-1
- Map Reduce
- Map Operator Tree:
- TableScan
- alias: t8
- Statistics: Num rows: 40/40 Data size: 320 Basic stats: COMPLETE Column stats: COMPLETE
- Filter Operator
- predicate: (b = 2) (type: boolean)
- Statistics: Num rows: 8/8 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE
- Select Operator
- expressions: a (type: int)
- outputColumnNames: _col1
- Statistics: Num rows: 8/8 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE
- Group By Operator
- aggregations: sum(_col1)
- keys: 2 (type: int)
- mode: hash
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 1/1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE
- Reduce Output Operator
- key expressions: _col0 (type: int)
- sort order: +
- Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 1/1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE
- value expressions: _col1 (type: bigint)
- Execution mode: vectorized
- Reduce Operator Tree:
- Group By Operator
- aggregations: sum(VALUE._col0)
- keys: KEY._col0 (type: int)
- mode: mergepartial
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 1/1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE
- Select Operator
- expressions: _col1 (type: bigint)
- outputColumnNames: _col0
- Statistics: Num rows: 1/1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
- File Output Operator
- compressed: false
- Statistics: Num rows: 1/1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-
- Stage: Stage-0
- Fetch Operator
- limit: -1
- Processor Tree:
- ListSink
-
-PREHOOK: query: select sum(a) from t1 where 2=b and b=2 group by b
-PREHOOK: type: QUERY
-PREHOOK: Input: default@t1
-#### A masked pattern was here ####
-POSTHOOK: query: select sum(a) from t1 where 2=b and b=2 group by b
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@t1
-#### A masked pattern was here ####
-PREHOOK: query: explain analyze select sum(a) from t1 where 2=b and b=2 group by b
-PREHOOK: type: QUERY
-POSTHOOK: query: explain analyze select sum(a) from t1 where 2=b and b=2 group by b
-POSTHOOK: type: QUERY
-STAGE DEPENDENCIES:
- Stage-1 is a root stage
- Stage-0 depends on stages: Stage-1
-
-STAGE PLANS:
- Stage: Stage-1
- Map Reduce
- Map Operator Tree:
- TableScan
- alias: t1
- Statistics: Num rows: 5/5 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE
- Filter Operator
- predicate: (b = 2) (type: boolean)
- Statistics: Num rows: 1/1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
- Select Operator
- expressions: a (type: int)
- outputColumnNames: _col1
- Statistics: Num rows: 1/1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
- Group By Operator
- aggregations: sum(_col1)
- keys: 2 (type: int)
- mode: hash
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 1/1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE
- Reduce Output Operator
- key expressions: _col0 (type: int)
- sort order: +
- Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 1/1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE
- value expressions: _col1 (type: bigint)
- Execution mode: vectorized
- Reduce Operator Tree:
- Group By Operator
- aggregations: sum(VALUE._col0)
- keys: KEY._col0 (type: int)
- mode: mergepartial
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 1/1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE
- Select Operator
- expressions: _col1 (type: bigint)
- outputColumnNames: _col0
- Statistics: Num rows: 1/1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
- File Output Operator
- compressed: false
- Statistics: Num rows: 1/1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-
- Stage: Stage-0
- Fetch Operator
- limit: -1
- Processor Tree:
- ListSink
-
-PREHOOK: query: select sum(a) from t1 where b in (2,3) and b=2 group by b
-PREHOOK: type: QUERY
-PREHOOK: Input: default@t1
-#### A masked pattern was here ####
-POSTHOOK: query: select sum(a) from t1 where b in (2,3) and b=2 group by b
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@t1
-#### A masked pattern was here ####
-PREHOOK: query: explain analyze select sum(a) from t1 where b in (2,3) and b=2 group by b
-PREHOOK: type: QUERY
-POSTHOOK: query: explain analyze select sum(a) from t1 where b in (2,3) and b=2 group by b
-POSTHOOK: type: QUERY
-STAGE DEPENDENCIES:
- Stage-1 is a root stage
- Stage-0 depends on stages: Stage-1
-
-STAGE PLANS:
- Stage: Stage-1
- Map Reduce
- Map Operator Tree:
- TableScan
- alias: t1
- Statistics: Num rows: 5/5 Data size: 40 Basic stats: COMPLETE Column stats: COMPLETE
- Filter Operator
- predicate: ((b = 2) and (b) IN (2, 3)) (type: boolean)
- Statistics: Num rows: 1/1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
- Select Operator
- expressions: a (type: int)
- outputColumnNames: _col1
- Statistics: Num rows: 1/1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
- Group By Operator
- aggregations: sum(_col1)
- keys: 2 (type: int)
- mode: hash
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 1/1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE
- Reduce Output Operator
- key expressions: _col0 (type: int)
- sort order: +
- Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 1/1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE
- value expressions: _col1 (type: bigint)
- Execution mode: vectorized
- Reduce Operator Tree:
- Group By Operator
- aggregations: sum(VALUE._col0)
- keys: KEY._col0 (type: int)
- mode: mergepartial
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 1/1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE
- Select Operator
- expressions: _col1 (type: bigint)
- outputColumnNames: _col0
- Statistics: Num rows: 1/1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
- File Output Operator
- compressed: false
- Statistics: Num rows: 1/1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-
- Stage: Stage-0
- Fetch Operator
- limit: -1
- Processor Tree:
- ListSink
-
-PREHOOK: query: select sum(a) from t8 where b in (2,3) and b=2 group by b
-PREHOOK: type: QUERY
-PREHOOK: Input: default@t8
-#### A masked pattern was here ####
-POSTHOOK: query: select sum(a) from t8 where b in (2,3) and b=2 group by b
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@t8
-#### A masked pattern was here ####
-PREHOOK: query: explain analyze select sum(a) from t8 where b in (2,3) and b=2 group by b
-PREHOOK: type: QUERY
-POSTHOOK: query: explain analyze select sum(a) from t8 where b in (2,3) and b=2 group by b
-POSTHOOK: type: QUERY
-STAGE DEPENDENCIES:
- Stage-1 is a root stage
- Stage-0 depends on stages: Stage-1
-
-STAGE PLANS:
- Stage: Stage-1
- Map Reduce
- Map Operator Tree:
- TableScan
- alias: t8
- Statistics: Num rows: 40/40 Data size: 320 Basic stats: COMPLETE Column stats: COMPLETE
- Filter Operator
- predicate: ((b = 2) and (b) IN (2, 3)) (type: boolean)
- Statistics: Num rows: 8/8 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE
- Select Operator
- expressions: a (type: int)
- outputColumnNames: _col1
- Statistics: Num rows: 8/8 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE
- Group By Operator
- aggregations: sum(_col1)
- keys: 2 (type: int)
- mode: hash
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 1/1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE
- Reduce Output Operator
- key expressions: _col0 (type: int)
- sort order: +
- Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 1/1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE
- value expressions: _col1 (type: bigint)
- Execution mode: vectorized
- Reduce Operator Tree:
- Group By Operator
- aggregations: sum(VALUE._col0)
- keys: KEY._col0 (type: int)
- mode: mergepartial
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 1/1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE
- Select Operator
- expressions: _col1 (type: bigint)
- outputColumnNames: _col0
- Statistics: Num rows: 1/1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
- File Output Operator
- compressed: false
- Statistics: Num rows: 1/1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-
- Stage: Stage-0
- Fetch Operator
- limit: -1
- Processor Tree:
- ListSink
-
-Warning: Shuffle Join JOIN[8][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product
-PREHOOK: query: select count(*) from t8 ta, t8 tb where ta.a = tb.b and ta.a=3
-PREHOOK: type: QUERY
-PREHOOK: Input: default@t8
-#### A masked pattern was here ####
-POSTHOOK: query: select count(*) from t8 ta, t8 tb where ta.a = tb.b and ta.a=3
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@t8
-#### A masked pattern was here ####
-Warning: Shuffle Join JOIN[8][tables = [$hdt$_0, $hdt$_1]] in Stage 'Stage-1:MAPRED' is a cross product
-PREHOOK: query: explain analyze select count(*) from t8 ta, t8 tb where ta.a = tb.b and ta.a=3
-PREHOOK: type: QUERY
-POSTHOOK: query: explain analyze select count(*) from t8 ta, t8 tb where ta.a = tb.b and ta.a=3
-POSTHOOK: type: QUERY
-STAGE DEPENDENCIES:
- Stage-1 is a root stage
- Stage-2 depends on stages: Stage-1
- Stage-0 depends on stages: Stage-2
-
-STAGE PLANS:
- Stage: Stage-1
- Map Reduce
- Map Operator Tree:
- TableScan
- alias: ta
- Statistics: Num rows: 40/40 Data size: 160 Basic stats: COMPLETE Column stats: COMPLETE
- Filter Operator
- predicate: (a = 3) (type: boolean)
- Statistics: Num rows: 8/8 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE
- Select Operator
- Statistics: Num rows: 8/8 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE
- Reduce Output Operator
- sort order:
- Statistics: Num rows: 8/8 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE
- TableScan
- alias: tb
- Statistics: Num rows: 40/40 Data size: 160 Basic stats: COMPLETE Column stats: COMPLETE
- Filter Operator
- predicate: (b = 3) (type: boolean)
- Statistics: Num rows: 8/8 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE
- Select Operator
- Statistics: Num rows: 8/8 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE
- Reduce Output Operator
- sort order:
- Statistics: Num rows: 8/8 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE
- Reduce Operator Tree:
- Join Operator
- condition map:
- Inner Join 0 to 1
- keys:
- 0
- 1
- Statistics: Num rows: 64/64 Data size: 512 Basic stats: COMPLETE Column stats: COMPLETE
- Group By Operator
- aggregations: count()
- mode: hash
- outputColumnNames: _col0
- Statistics: Num rows: 1/1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
- File Output Operator
- compressed: false
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
-
- Stage: Stage-2
- Map Reduce
- Map Operator Tree:
- TableScan
- Reduce Output Operator
- sort order:
- Statistics: Num rows: 1/1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
- value expressions: _col0 (type: bigint)
- Execution mode: vectorized
- Reduce Operator Tree:
- Group By Operator
- aggregations: count(VALUE._col0)
- mode: mergepartial
- outputColumnNames: _col0
- Statistics: Num rows: 1/1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
- File Output Operator
- compressed: false
- Statistics: Num rows: 1/1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-
- Stage: Stage-0
- Fetch Operator
- limit: -1
- Processor Tree:
- ListSink
-
-PREHOOK: query: select sum(a) from t8 where b in (1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50) and b=2 and b=2 and 2=b group by b
-PREHOOK: type: QUERY
-PREHOOK: Input: default@t8
-#### A masked pattern was here ####
-POSTHOOK: query: select sum(a) from t8 where b in (1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50) and b=2 and b=2 and 2=b group by b
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@t8
-#### A masked pattern was here ####
-PREHOOK: query: explain analyze select sum(a) from t8 where b in (1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50) and b=2 and b=2 and 2=b group by b
-PREHOOK: type: QUERY
-POSTHOOK: query: explain analyze select sum(a) from t8 where b in (1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50) and b=2 and b=2 and 2=b group by b
-POSTHOOK: type: QUERY
-STAGE DEPENDENCIES:
- Stage-1 is a root stage
- Stage-0 depends on stages: Stage-1
-
-STAGE PLANS:
- Stage: Stage-1
- Map Reduce
- Map Operator Tree:
- TableScan
- alias: t8
- Statistics: Num rows: 40/40 Data size: 320 Basic stats: COMPLETE Column stats: COMPLETE
- Filter Operator
- predicate: ((b = 2) and (b) IN (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50)) (type: boolean)
- Statistics: Num rows: 8/8 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE
- Select Operator
- expressions: a (type: int)
- outputColumnNames: _col1
- Statistics: Num rows: 8/8 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE
- Group By Operator
- aggregations: sum(_col1)
- keys: 2 (type: int)
- mode: hash
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 1/1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE
- Reduce Output Operator
- key expressions: _col0 (type: int)
- sort order: +
- Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 1/1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE
- value expressions: _col1 (type: bigint)
- Execution mode: vectorized
- Reduce Operator Tree:
- Group By Operator
- aggregations: sum(VALUE._col0)
- keys: KEY._col0 (type: int)
- mode: mergepartial
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 1/1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE
- Select Operator
- expressions: _col1 (type: bigint)
- outputColumnNames: _col0
- Statistics: Num rows: 1/1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
- File Output Operator
- compressed: false
- Statistics: Num rows: 1/1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-
- Stage: Stage-0
- Fetch Operator
- limit: -1
- Processor Tree:
- ListSink
-
-PREHOOK: query: select sum(a) from t8 where b=2 and (b = 1 or b=2) group by b
-PREHOOK: type: QUERY
-PREHOOK: Input: default@t8
-#### A masked pattern was here ####
-POSTHOOK: query: select sum(a) from t8 where b=2 and (b = 1 or b=2) group by b
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@t8
-#### A masked pattern was here ####
-PREHOOK: query: explain analyze select sum(a) from t8 where b=2 and (b = 1 or b=2) group by b
-PREHOOK: type: QUERY
-POSTHOOK: query: explain analyze select sum(a) from t8 where b=2 and (b = 1 or b=2) group by b
-POSTHOOK: type: QUERY
-STAGE DEPENDENCIES:
- Stage-1 is a root stage
- Stage-0 depends on stages: Stage-1
-
-STAGE PLANS:
- Stage: Stage-1
- Map Reduce
- Map Operator Tree:
- TableScan
- alias: t8
- Statistics: Num rows: 40/40 Data size: 320 Basic stats: COMPLETE Column stats: COMPLETE
- Filter Operator
- predicate: (((b = 1) or (b = 2)) and (b = 2)) (type: boolean)
- Statistics: Num rows: 8/8 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE
- Select Operator
- expressions: a (type: int)
- outputColumnNames: _col1
- Statistics: Num rows: 8/8 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE
- Group By Operator
- aggregations: sum(_col1)
- keys: 2 (type: int)
- mode: hash
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 1/1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE
- Reduce Output Operator
- key expressions: _col0 (type: int)
- sort order: +
- Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 1/1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE
- value expressions: _col1 (type: bigint)
- Execution mode: vectorized
- Reduce Operator Tree:
- Group By Operator
- aggregations: sum(VALUE._col0)
- keys: KEY._col0 (type: int)
- mode: mergepartial
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 1/1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE
- Select Operator
- expressions: _col1 (type: bigint)
- outputColumnNames: _col0
- Statistics: Num rows: 1/1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
- File Output Operator
- compressed: false
- Statistics: Num rows: 1/1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-
- Stage: Stage-0
- Fetch Operator
- limit: -1
- Processor Tree:
- ListSink
-
-PREHOOK: query: select sum(a) from t8 where b=2 and (b = 1 or b=2) and (b=1 or b=3) group by b
-PREHOOK: type: QUERY
-PREHOOK: Input: default@t8
-#### A masked pattern was here ####
-POSTHOOK: query: select sum(a) from t8 where b=2 and (b = 1 or b=2) and (b=1 or b=3) group by b
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@t8
-#### A masked pattern was here ####
-PREHOOK: query: explain analyze select sum(a) from t8 where b=2 and (b = 1 or b=2) and (b=1 or b=3) group by b
-PREHOOK: type: QUERY
-POSTHOOK: query: explain analyze select sum(a) from t8 where b=2 and (b = 1 or b=2) and (b=1 or b=3) group by b
-POSTHOOK: type: QUERY
-STAGE DEPENDENCIES:
- Stage-1 is a root stage
- Stage-0 depends on stages: Stage-1
-
-STAGE PLANS:
- Stage: Stage-1
- Map Reduce
- Map Operator Tree:
- TableScan
- alias: t8
- Statistics: Num rows: 40/40 Data size: 320 Basic stats: COMPLETE Column stats: COMPLETE
- Filter Operator
- predicate: (((b = 1) or (b = 2)) and ((b = 1) or (b = 3)) and (b = 2)) (type: boolean)
- Statistics: Num rows: 8/0 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE
- Select Operator
- expressions: a (type: int)
- outputColumnNames: _col1
- Statistics: Num rows: 8/0 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE
- Group By Operator
- aggregations: sum(_col1)
- keys: 2 (type: int)
- mode: hash
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 1/0 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE
- Reduce Output Operator
- key expressions: _col0 (type: int)
- sort order: +
- Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 1/0 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE
- value expressions: _col1 (type: bigint)
- Execution mode: vectorized
- Reduce Operator Tree:
- Group By Operator
- aggregations: sum(VALUE._col0)
- keys: KEY._col0 (type: int)
- mode: mergepartial
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 1/0 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE
- Select Operator
- expressions: _col1 (type: bigint)
- outputColumnNames: _col0
- Statistics: Num rows: 1/0 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
- File Output Operator
- compressed: false
- Statistics: Num rows: 1/0 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-
- Stage: Stage-0
- Fetch Operator
- limit: -1
- Processor Tree:
- ListSink
-
-PREHOOK: query: select sum(a) from t8 where
- b=2 and (b = 1 or b=2)
-and
- a=3 and (a = 3 or a=4)
-group by b
-PREHOOK: type: QUERY
-PREHOOK: Input: default@t8
-#### A masked pattern was here ####
-POSTHOOK: query: select sum(a) from t8 where
- b=2 and (b = 1 or b=2)
-and
- a=3 and (a = 3 or a=4)
-group by b
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@t8
-#### A masked pattern was here ####
-PREHOOK: query: explain analyze select sum(a) from t8 where
- b=2 and (b = 1 or b=2)
-and
- a=3 and (a = 3 or a=4)
-group by b
-PREHOOK: type: QUERY
-POSTHOOK: query: explain analyze select sum(a) from t8 where
- b=2 and (b = 1 or b=2)
-and
- a=3 and (a = 3 or a=4)
-group by b
-POSTHOOK: type: QUERY
-STAGE DEPENDENCIES:
- Stage-1 is a root stage
- Stage-0 depends on stages: Stage-1
-
-STAGE PLANS:
- Stage: Stage-1
- Map Reduce
- Map Operator Tree:
- TableScan
- alias: t8
- Statistics: Num rows: 40/40 Data size: 320 Basic stats: COMPLETE Column stats: COMPLETE
- Filter Operator
- predicate: (((a = 3) or (a = 4)) and ((b = 1) or (b = 2)) and (a = 3) and (b = 2)) (type: boolean)
- Statistics: Num rows: 8/0 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE
- Select Operator
- Statistics: Num rows: 8/0 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE
- Group By Operator
- aggregations: sum(3)
- keys: 2 (type: int)
- mode: hash
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 1/0 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE
- Reduce Output Operator
- key expressions: _col0 (type: int)
- sort order: +
- Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 1/0 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE
- value expressions: _col1 (type: bigint)
- Execution mode: vectorized
- Reduce Operator Tree:
- Group By Operator
- aggregations: sum(VALUE._col0)
- keys: KEY._col0 (type: int)
- mode: mergepartial
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 1/0 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE
- Select Operator
- expressions: _col1 (type: bigint)
- outputColumnNames: _col0
- Statistics: Num rows: 1/0 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
- File Output Operator
- compressed: false
- Statistics: Num rows: 1/0 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-
- Stage: Stage-0
- Fetch Operator
- limit: -1
- Processor Tree:
- ListSink
-
[4/4] hive git commit: Revert "HIVE-19500: Prevent multiple
selectivity estimations for the same variable in conjuctions (Zoltan
Haindrich reviewed by Ashutosh Chauhan)"
Posted by jc...@apache.org.
Revert "HIVE-19500: Prevent multiple selectivity estimations for the same variable in conjuctions (Zoltan Haindrich reviewed by Ashutosh Chauhan)"
This reverts commit 24d70279ee332fed24272d19555c8c19887230b2.
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/a1e6206c
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/a1e6206c
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/a1e6206c
Branch: refs/heads/master
Commit: a1e6206cc4f77c20990c9f2436fa5940ec6a7e3c
Parents: 7056445
Author: Jesus Camacho Rodriguez <jc...@apache.org>
Authored: Wed May 23 09:52:46 2018 -0700
Committer: Jesus Camacho Rodriguez <jc...@apache.org>
Committed: Wed May 23 09:52:46 2018 -0700
----------------------------------------------------------------------
.../stats/annotation/StatsRulesProcFactory.java | 12 +-
.../clientpositive/stat_estimate_related_col.q | 43 -
.../annotate_stats_deep_filters.q.out | 4 +-
.../annotate_stats_join_pkfk.q.out | 48 +-
.../llap/bucketsortoptimize_insert_7.q.out | 20 +-
.../clientpositive/llap/explainuser_1.q.out | 64 +-
.../llap/orc_predicate_pushdown.q.out | 20 +-
.../llap/parquet_predicate_pushdown.q.out | 20 +-
.../clientpositive/llap/subquery_multi.q.out | 26 +-
.../llap/vector_decimal_cast.q.out | 4 +-
.../clientpositive/llap/vector_if_expr.q.out | 10 +-
.../llap/vector_mapjoin_reduce.q.out | 14 +-
.../clientpositive/llap/vectorization_0.q.out | 12 +-
.../llap/vectorization_short_regress.q.out | 8 +-
.../llap/vectorized_nested_mapjoin.q.out | 12 +-
.../spark/spark_explainuser_1.q.out | 68 +-
.../stat_estimate_related_col.q.out | 784 -------------------
17 files changed, 171 insertions(+), 998 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/a1e6206c/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java
index c770227..345595b 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java
@@ -347,12 +347,12 @@ public class StatsRulesProcFactory {
for (ExprNodeDesc child : genFunc.getChildren()) {
evaluatedRowCount = evaluateChildExpr(aspCtx.getAndExprStats(), child,
aspCtx, neededCols, op, evaluatedRowCount);
- newNumRows = evaluatedRowCount;
- if (satisfyPrecondition(aspCtx.getAndExprStats())) {
- updateStats(aspCtx.getAndExprStats(), newNumRows, true, op);
- } else {
- updateStats(aspCtx.getAndExprStats(), newNumRows, false, op);
- }
+ }
+ newNumRows = evaluatedRowCount;
+ if (satisfyPrecondition(aspCtx.getAndExprStats())) {
+ updateStats(aspCtx.getAndExprStats(), newNumRows, true, op);
+ } else {
+ updateStats(aspCtx.getAndExprStats(), newNumRows, false, op);
}
} else if (udf instanceof GenericUDFOPOr) {
// for OR condition independently compute and update stats.
http://git-wip-us.apache.org/repos/asf/hive/blob/a1e6206c/ql/src/test/queries/clientpositive/stat_estimate_related_col.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/stat_estimate_related_col.q b/ql/src/test/queries/clientpositive/stat_estimate_related_col.q
deleted file mode 100644
index 52da2f7..0000000
--- a/ql/src/test/queries/clientpositive/stat_estimate_related_col.q
+++ /dev/null
@@ -1,43 +0,0 @@
-set hive.explain.user=true;
-set hive.strict.checks.cartesian.product=false;
-set hive.stats.fetch.column.stats=true;
-
-drop table if exists t1;
-drop table if exists t8;
-
-create table t1 (a integer,b integer);
-create table t8 like t1;
-
-insert into t1 values (1,1),(2,2),(3,3),(4,4),(5,5);
-
-insert into t8
-select * from t1 union all select * from t1 union all select * from t1 union all select * from t1 union all
-select * from t1 union all select * from t1 union all select * from t1 union all select * from t1
-;
-
-analyze table t1 compute statistics for columns;
-analyze table t8 compute statistics for columns;
-
-explain analyze select sum(a) from t8 where b in (2,3) group by b;
-explain analyze select sum(a) from t8 where b=2 group by b;
-
-explain analyze select sum(a) from t1 where 2=b and b=2 group by b;
-
-explain analyze select sum(a) from t1 where b in (2,3) and b=2 group by b;
-explain analyze select sum(a) from t8 where b in (2,3) and b=2 group by b;
-
-
-explain analyze select count(*) from t8 ta, t8 tb where ta.a = tb.b and ta.a=3;
-
-
-explain analyze select sum(a) from t8 where b in (1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50) and b=2 and b=2 and 2=b group by b;
-
-explain analyze select sum(a) from t8 where b=2 and (b = 1 or b=2) group by b;
-explain analyze select sum(a) from t8 where b=2 and (b = 1 or b=2) and (b=1 or b=3) group by b;
-
-explain analyze select sum(a) from t8 where
- b=2 and (b = 1 or b=2)
-and
- a=3 and (a = 3 or a=4)
-group by b;
-
http://git-wip-us.apache.org/repos/asf/hive/blob/a1e6206c/ql/src/test/results/clientpositive/annotate_stats_deep_filters.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/annotate_stats_deep_filters.q.out b/ql/src/test/results/clientpositive/annotate_stats_deep_filters.q.out
index 5f68009..dfdfe39 100644
--- a/ql/src/test/results/clientpositive/annotate_stats_deep_filters.q.out
+++ b/ql/src/test/results/clientpositive/annotate_stats_deep_filters.q.out
@@ -123,9 +123,9 @@ STAGE PLANS:
Statistics: Num rows: 2098 Data size: 16744 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
predicate: (((t = 10Y) and (si = 11S)) or ((t = 11Y) and (si = 12S)) or ((t = 12Y) and (si = 13S)) or ((t = 13Y) and (si = 14S)) or ((t = 14Y) and (si = 15S)) or ((t = 15Y) and (si = 16S)) or ((t = 16Y) and (si = 17S)) or ((t = 17Y) and (si = 18S)) or ((t = 1Y) and (si = 2S)) or ((t = 27Y) and (si = 28S)) or ((t = 2Y) and (si = 3S)) or ((t = 37Y) and (si = 38S)) or ((t = 3Y) and (si = 4S)) or ((t = 47Y) and (si = 48S)) or ((t = 4Y) and (si = 5S)) or ((t = 52Y) and (si = 53S)) or ((t = 5Y) and (si = 6S)) or ((t = 6Y) and (si = 7S)) or ((t = 7Y) and (si = 8S)) or ((t = 9Y) and (si = 10S))) (type: boolean)
- Statistics: Num rows: 160 Data size: 1280 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
- Statistics: Num rows: 160 Data size: 1280 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
aggregations: count()
mode: hash
http://git-wip-us.apache.org/repos/asf/hive/blob/a1e6206c/ql/src/test/results/clientpositive/annotate_stats_join_pkfk.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/annotate_stats_join_pkfk.q.out b/ql/src/test/results/clientpositive/annotate_stats_join_pkfk.q.out
index d9f6bad..aa5dba0 100644
--- a/ql/src/test/results/clientpositive/annotate_stats_join_pkfk.q.out
+++ b/ql/src/test/results/clientpositive/annotate_stats_join_pkfk.q.out
@@ -585,16 +585,16 @@ STAGE PLANS:
Statistics: Num rows: 1000 Data size: 7676 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
predicate: ((ss_quantity > 10) and ss_store_sk is not null) (type: boolean)
- Statistics: Num rows: 321 Data size: 2468 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 297 Data size: 2284 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: ss_store_sk (type: int)
outputColumnNames: _col0
- Statistics: Num rows: 321 Data size: 2468 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 297 Data size: 2284 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 321 Data size: 2468 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 297 Data size: 2284 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Operator Tree:
Join Operator
condition map:
@@ -603,10 +603,10 @@ STAGE PLANS:
0 _col0 (type: int)
1 _col0 (type: int)
outputColumnNames: _col0
- Statistics: Num rows: 321 Data size: 1284 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 297 Data size: 1188 Basic stats: COMPLETE Column stats: PARTIAL
File Output Operator
compressed: false
- Statistics: Num rows: 321 Data size: 1284 Basic stats: COMPLETE Column stats: PARTIAL
+ Statistics: Num rows: 297 Data size: 1188 Basic stats: COMPLETE Column stats: PARTIAL
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -715,16 +715,16 @@ STAGE PLANS:
Statistics: Num rows: 1000 Data size: 7676 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
predicate: ((ss_quantity > 10) and ss_store_sk is not null) (type: boolean)
- Statistics: Num rows: 321 Data size: 2468 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 297 Data size: 2284 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: ss_store_sk (type: int)
outputColumnNames: _col0
- Statistics: Num rows: 321 Data size: 2468 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 297 Data size: 2284 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 321 Data size: 2468 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 297 Data size: 2284 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Operator Tree:
Join Operator
condition map:
@@ -733,10 +733,10 @@ STAGE PLANS:
0 _col0 (type: int)
1 _col0 (type: int)
outputColumnNames: _col0
- Statistics: Num rows: 321 Data size: 1284 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 297 Data size: 1188 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
- Statistics: Num rows: 321 Data size: 1284 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 297 Data size: 1188 Basic stats: COMPLETE Column stats: COMPLETE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -1023,16 +1023,16 @@ STAGE PLANS:
Statistics: Num rows: 1000 Data size: 7676 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
predicate: ((ss_quantity > 10) and ss_store_sk is not null) (type: boolean)
- Statistics: Num rows: 321 Data size: 2468 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 297 Data size: 2284 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: ss_store_sk (type: int)
outputColumnNames: _col0
- Statistics: Num rows: 321 Data size: 2468 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 297 Data size: 2284 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 321 Data size: 2468 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 297 Data size: 2284 Basic stats: COMPLETE Column stats: COMPLETE
TableScan
alias: s
Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE
@@ -1073,14 +1073,14 @@ STAGE PLANS:
1 _col0 (type: int)
2 _col0 (type: int)
outputColumnNames: _col2
- Statistics: Num rows: 321 Data size: 1284 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 297 Data size: 1188 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: _col2 (type: int)
outputColumnNames: _col0
- Statistics: Num rows: 321 Data size: 1284 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 297 Data size: 1188 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
- Statistics: Num rows: 321 Data size: 1284 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 297 Data size: 1188 Basic stats: COMPLETE Column stats: COMPLETE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -1110,16 +1110,16 @@ STAGE PLANS:
Statistics: Num rows: 1000 Data size: 7664 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
predicate: (ss_addr_sk is not null and ss_store_sk is not null) (type: boolean)
- Statistics: Num rows: 916 Data size: 7020 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 914 Data size: 7004 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: ss_addr_sk (type: int), ss_store_sk (type: int)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 916 Data size: 7020 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 914 Data size: 7004 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col1 (type: int)
sort order: +
Map-reduce partition columns: _col1 (type: int)
- Statistics: Num rows: 916 Data size: 7020 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 914 Data size: 7004 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col0 (type: int)
TableScan
alias: s
@@ -1144,7 +1144,7 @@ STAGE PLANS:
0 _col1 (type: int)
1 _col0 (type: int)
outputColumnNames: _col0, _col2
- Statistics: Num rows: 916 Data size: 7148 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 914 Data size: 7132 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
table:
@@ -1160,7 +1160,7 @@ STAGE PLANS:
key expressions: _col0 (type: int)
sort order: +
Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 916 Data size: 7148 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 914 Data size: 7132 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col2 (type: int)
TableScan
alias: ca
@@ -1185,14 +1185,14 @@ STAGE PLANS:
0 _col0 (type: int)
1 _col0 (type: int)
outputColumnNames: _col2
- Statistics: Num rows: 241 Data size: 964 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 243 Data size: 972 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: _col2 (type: int)
outputColumnNames: _col0
- Statistics: Num rows: 241 Data size: 964 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 243 Data size: 972 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
- Statistics: Num rows: 241 Data size: 964 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 243 Data size: 972 Basic stats: COMPLETE Column stats: COMPLETE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/a1e6206c/ql/src/test/results/clientpositive/llap/bucketsortoptimize_insert_7.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/bucketsortoptimize_insert_7.q.out b/ql/src/test/results/clientpositive/llap/bucketsortoptimize_insert_7.q.out
index cf7e442..0246833 100644
--- a/ql/src/test/results/clientpositive/llap/bucketsortoptimize_insert_7.q.out
+++ b/ql/src/test/results/clientpositive/llap/bucketsortoptimize_insert_7.q.out
@@ -518,22 +518,22 @@ STAGE PLANS:
Statistics: Num rows: 84 Data size: 7896 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
predicate: (((key = 0) or (key = 5)) and (key < 8)) (type: boolean)
- Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: key (type: int), value (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 2 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE
Map Operator Tree:
TableScan
alias: test_table1_n20
Statistics: Num rows: 10 Data size: 930 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
predicate: (((key = 0) or (key = 5)) and (key < 8)) (type: boolean)
- Statistics: Num rows: 3 Data size: 279 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 2 Data size: 186 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: key (type: int), value (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 3 Data size: 279 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 2 Data size: 186 Basic stats: COMPLETE Column stats: COMPLETE
Merge Join Operator
condition map:
Inner Join 0 to 1
@@ -541,16 +541,16 @@ STAGE PLANS:
0 _col0 (type: int)
1 _col0 (type: int)
outputColumnNames: _col0, _col1, _col3
- Statistics: Num rows: 3 Data size: 549 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 183 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: _col0 (type: int), concat(_col1, _col3) (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 3 Data size: 564 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col0 (type: int)
sort order: +
Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 3 Data size: 564 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col1 (type: string)
Execution mode: llap
Reducer 2
@@ -559,10 +559,10 @@ STAGE PLANS:
Select Operator
expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 3 Data size: 564 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
- Statistics: Num rows: 3 Data size: 564 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: COMPLETE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
@@ -571,7 +571,7 @@ STAGE PLANS:
Select Operator
expressions: _col0 (type: int), _col1 (type: string), '1' (type: string)
outputColumnNames: key, value, ds
- Statistics: Num rows: 3 Data size: 819 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 273 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
aggregations: compute_stats(key, 'hll'), compute_stats(value, 'hll')
keys: ds (type: string)
http://git-wip-us.apache.org/repos/asf/hive/blob/a1e6206c/ql/src/test/results/clientpositive/llap/explainuser_1.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/explainuser_1.q.out b/ql/src/test/results/clientpositive/llap/explainuser_1.q.out
index 0c339e5..f4852d2 100644
--- a/ql/src/test/results/clientpositive/llap/explainuser_1.q.out
+++ b/ql/src/test/results/clientpositive/llap/explainuser_1.q.out
@@ -419,25 +419,25 @@ Stage-0
<-Map 5 [SIMPLE_EDGE] llap
SHUFFLE [RS_7]
PartitionCols:_col0, _col1, _col2
- Group By Operator [GBY_6] (rows=2 width=101)
+ Group By Operator [GBY_6] (rows=1 width=101)
Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(c_int)"],keys:key, c_int, c_float
- Filter Operator [FIL_37] (rows=5 width=93)
+ Filter Operator [FIL_37] (rows=4 width=93)
predicate:(((c_int + 1) >= 0) and ((c_int > 0) or (c_float >= 0)) and key is not null)
TableScan [TS_3] (rows=20 width=88)
default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"]
<-Reducer 8 [SIMPLE_EDGE] llap
SHUFFLE [RS_19]
PartitionCols:_col0
- Select Operator [SEL_16] (rows=2 width=89)
+ Select Operator [SEL_16] (rows=1 width=89)
Output:["_col0","_col1"]
- Group By Operator [GBY_15] (rows=2 width=93)
+ Group By Operator [GBY_15] (rows=1 width=93)
Output:["_col0","_col1","_col2"],keys:KEY._col0, KEY._col1, KEY._col2
<-Map 7 [SIMPLE_EDGE] llap
SHUFFLE [RS_14]
PartitionCols:_col0, _col1, _col2
- Group By Operator [GBY_13] (rows=2 width=93)
+ Group By Operator [GBY_13] (rows=1 width=93)
Output:["_col0","_col1","_col2"],keys:key, c_int, c_float
- Filter Operator [FIL_38] (rows=5 width=93)
+ Filter Operator [FIL_38] (rows=4 width=93)
predicate:(((c_int + 1) >= 0) and ((c_int > 0) or (c_float >= 0)) and key is not null)
TableScan [TS_10] (rows=20 width=88)
default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"]
@@ -1119,25 +1119,25 @@ Stage-0
Stage-1
Reducer 2 llap
File Output Operator [FS_14]
- Select Operator [SEL_13] (rows=40 width=101)
+ Select Operator [SEL_13] (rows=48 width=101)
Output:["_col0","_col1","_col2","_col3","_col4"]
- Merge Join Operator [MERGEJOIN_24] (rows=40 width=101)
+ Merge Join Operator [MERGEJOIN_24] (rows=48 width=101)
Conds:RS_9._col0=RS_10._col0(Inner),RS_9._col0=RS_11._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col6"],residual filter predicates:{((_col1 + _col4) = 2)} {((_col1 > 0) or (_col6 >= 0))}
<-Map 1 [SIMPLE_EDGE] llap
SHUFFLE [RS_9]
PartitionCols:_col0
- Select Operator [SEL_2] (rows=9 width=93)
+ Select Operator [SEL_2] (rows=8 width=93)
Output:["_col0","_col1","_col2"]
- Filter Operator [FIL_21] (rows=9 width=93)
+ Filter Operator [FIL_21] (rows=8 width=93)
predicate:(((c_int + 1) = 2) and ((c_int > 0) or (c_float >= 0)) and key is not null)
TableScan [TS_0] (rows=20 width=88)
default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"]
<-Map 3 [SIMPLE_EDGE] llap
SHUFFLE [RS_10]
PartitionCols:_col0
- Select Operator [SEL_5] (rows=9 width=89)
+ Select Operator [SEL_5] (rows=8 width=89)
Output:["_col0","_col1"]
- Filter Operator [FIL_22] (rows=9 width=93)
+ Filter Operator [FIL_22] (rows=8 width=93)
predicate:(((c_int + 1) = 2) and ((c_int > 0) or (c_float >= 0)) and key is not null)
TableScan [TS_3] (rows=20 width=88)
default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"]
@@ -1166,25 +1166,25 @@ Stage-0
Stage-1
Reducer 2 llap
File Output Operator [FS_14]
- Select Operator [SEL_13] (rows=40 width=101)
+ Select Operator [SEL_13] (rows=48 width=101)
Output:["_col0","_col1","_col2","_col3","_col4"]
- Merge Join Operator [MERGEJOIN_24] (rows=40 width=101)
+ Merge Join Operator [MERGEJOIN_24] (rows=48 width=101)
Conds:RS_9._col0=RS_10._col0(Inner),RS_9._col0=RS_11._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col6"],residual filter predicates:{((_col1 + _col4) = 2)} {((_col1 > 0) or (_col6 >= 0))}
<-Map 1 [SIMPLE_EDGE] llap
SHUFFLE [RS_9]
PartitionCols:_col0
- Select Operator [SEL_2] (rows=9 width=93)
+ Select Operator [SEL_2] (rows=8 width=93)
Output:["_col0","_col1","_col2"]
- Filter Operator [FIL_21] (rows=9 width=93)
+ Filter Operator [FIL_21] (rows=8 width=93)
predicate:(((c_int + 1) = 2) and ((c_int > 0) or (c_float >= 0)) and key is not null)
TableScan [TS_0] (rows=20 width=88)
default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"]
<-Map 3 [SIMPLE_EDGE] llap
SHUFFLE [RS_10]
PartitionCols:_col0
- Select Operator [SEL_5] (rows=9 width=89)
+ Select Operator [SEL_5] (rows=8 width=89)
Output:["_col0","_col1"]
- Filter Operator [FIL_22] (rows=9 width=93)
+ Filter Operator [FIL_22] (rows=8 width=93)
predicate:(((c_int + 1) = 2) and ((c_int > 0) or (c_float >= 0)) and key is not null)
TableScan [TS_3] (rows=20 width=88)
default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"]
@@ -1447,16 +1447,16 @@ Stage-0
Stage-1
Reducer 2 llap
File Output Operator [FS_12]
- Select Operator [SEL_11] (rows=9 width=4)
+ Select Operator [SEL_11] (rows=8 width=4)
Output:["_col0"]
- Merge Join Operator [MERGEJOIN_17] (rows=9 width=4)
+ Merge Join Operator [MERGEJOIN_17] (rows=8 width=4)
Conds:RS_8._col0=RS_9._col0(Left Semi),Output:["_col1"]
<-Map 1 [SIMPLE_EDGE] llap
SHUFFLE [RS_8]
PartitionCols:_col0
- Select Operator [SEL_2] (rows=9 width=93)
+ Select Operator [SEL_2] (rows=8 width=93)
Output:["_col0","_col1"]
- Filter Operator [FIL_15] (rows=9 width=93)
+ Filter Operator [FIL_15] (rows=8 width=93)
predicate:(((c_int + 1) = 2) and ((c_int > 0) or (c_float >= 0)) and key is not null)
TableScan [TS_0] (rows=20 width=88)
default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"]
@@ -1494,20 +1494,20 @@ Stage-0
<-Map 1 [SIMPLE_EDGE] llap
SHUFFLE [RS_13]
PartitionCols:_col0
- Select Operator [SEL_2] (rows=9 width=93)
+ Select Operator [SEL_2] (rows=8 width=93)
Output:["_col0","_col1","_col2"]
- Filter Operator [FIL_25] (rows=9 width=93)
+ Filter Operator [FIL_25] (rows=8 width=93)
predicate:(((c_int + 1) = 2) and ((c_int > 0) or (c_float >= 0)) and key is not null)
TableScan [TS_0] (rows=20 width=88)
default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"]
<-Map 3 [SIMPLE_EDGE] llap
SHUFFLE [RS_14]
PartitionCols:_col0
- Group By Operator [GBY_10] (rows=4 width=85)
+ Group By Operator [GBY_10] (rows=3 width=85)
Output:["_col0"],keys:_col0
- Select Operator [SEL_5] (rows=9 width=85)
+ Select Operator [SEL_5] (rows=8 width=85)
Output:["_col0"]
- Filter Operator [FIL_26] (rows=9 width=93)
+ Filter Operator [FIL_26] (rows=8 width=93)
predicate:(((c_int + 1) = 2) and ((c_int > 0) or (c_float >= 0)) and key is not null)
TableScan [TS_3] (rows=20 width=88)
default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"]
@@ -1992,18 +1992,18 @@ Stage-0
Stage-1
Reducer 4 llap
File Output Operator [FS_22]
- Select Operator [SEL_21] (rows=7 width=8)
+ Select Operator [SEL_21] (rows=3 width=8)
Output:["_col0","_col1"]
- Merge Join Operator [MERGEJOIN_32] (rows=7 width=8)
+ Merge Join Operator [MERGEJOIN_32] (rows=3 width=8)
Conds:RS_18._col1, _col4=RS_19._col0, _col1(Left Semi),Output:["_col0","_col3"]
<-Map 6 [SIMPLE_EDGE] llap
SHUFFLE [RS_19]
PartitionCols:_col0, _col1
- Group By Operator [GBY_17] (rows=4 width=8)
+ Group By Operator [GBY_17] (rows=1 width=8)
Output:["_col0","_col1"],keys:_col0, _col1
- Select Operator [SEL_12] (rows=14 width=8)
+ Select Operator [SEL_12] (rows=2 width=8)
Output:["_col0","_col1"]
- Filter Operator [FIL_30] (rows=14 width=96)
+ Filter Operator [FIL_30] (rows=2 width=96)
predicate:((l_linenumber = 1) and (l_shipmode = 'AIR') and l_orderkey is not null)
TableScan [TS_10] (rows=100 width=96)
default@lineitem,lineitem,Tbl:COMPLETE,Col:COMPLETE,Output:["l_orderkey","l_linenumber","l_shipmode"]
http://git-wip-us.apache.org/repos/asf/hive/blob/a1e6206c/ql/src/test/results/clientpositive/llap/orc_predicate_pushdown.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/orc_predicate_pushdown.q.out b/ql/src/test/results/clientpositive/llap/orc_predicate_pushdown.q.out
index e29c3d9..af7b0ba 100644
--- a/ql/src/test/results/clientpositive/llap/orc_predicate_pushdown.q.out
+++ b/ql/src/test/results/clientpositive/llap/orc_predicate_pushdown.q.out
@@ -592,15 +592,15 @@ STAGE PLANS:
Statistics: Num rows: 1049 Data size: 105941 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
predicate: ((not (t) IN (-1, -2, -3)) and (s like 'bob%') and s is not null and t BETWEEN 25 AND 30) (type: boolean)
- Statistics: Num rows: 55 Data size: 5555 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 56 Data size: 5656 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: t (type: tinyint), s (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 55 Data size: 5555 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 56 Data size: 5656 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col0 (type: tinyint), _col1 (type: string)
sort order: ++
- Statistics: Num rows: 55 Data size: 5555 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 56 Data size: 5656 Basic stats: COMPLETE Column stats: COMPLETE
Execution mode: llap
LLAP IO: all inputs
Reducer 2
@@ -609,10 +609,10 @@ STAGE PLANS:
Select Operator
expressions: KEY.reducesinkkey0 (type: tinyint), KEY.reducesinkkey1 (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 55 Data size: 5555 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 56 Data size: 5656 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
- Statistics: Num rows: 55 Data size: 5555 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 56 Data size: 5656 Basic stats: COMPLETE Column stats: COMPLETE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -658,15 +658,15 @@ STAGE PLANS:
Statistics: Num rows: 1049 Data size: 105941 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
predicate: ((not (t) IN (-1, -2, -3)) and (s like 'bob%') and s is not null and t BETWEEN 25 AND 30) (type: boolean)
- Statistics: Num rows: 55 Data size: 5555 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 56 Data size: 5656 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: t (type: tinyint), s (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 55 Data size: 5555 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 56 Data size: 5656 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col0 (type: tinyint), _col1 (type: string)
sort order: ++
- Statistics: Num rows: 55 Data size: 5555 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 56 Data size: 5656 Basic stats: COMPLETE Column stats: COMPLETE
Execution mode: llap
LLAP IO: all inputs
Reducer 2
@@ -675,10 +675,10 @@ STAGE PLANS:
Select Operator
expressions: KEY.reducesinkkey0 (type: tinyint), KEY.reducesinkkey1 (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 55 Data size: 5555 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 56 Data size: 5656 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
- Statistics: Num rows: 55 Data size: 5555 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 56 Data size: 5656 Basic stats: COMPLETE Column stats: COMPLETE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/a1e6206c/ql/src/test/results/clientpositive/llap/parquet_predicate_pushdown.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/parquet_predicate_pushdown.q.out b/ql/src/test/results/clientpositive/llap/parquet_predicate_pushdown.q.out
index eaa7e5d..84c5b2f 100644
--- a/ql/src/test/results/clientpositive/llap/parquet_predicate_pushdown.q.out
+++ b/ql/src/test/results/clientpositive/llap/parquet_predicate_pushdown.q.out
@@ -530,15 +530,15 @@ STAGE PLANS:
Statistics: Num rows: 1049 Data size: 105941 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
predicate: ((not (t) IN (-1, -2, -3)) and (s like 'bob%') and s is not null and t BETWEEN 25 AND 30) (type: boolean)
- Statistics: Num rows: 55 Data size: 5555 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 56 Data size: 5656 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: t (type: tinyint), s (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 55 Data size: 5555 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 56 Data size: 5656 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col0 (type: tinyint), _col1 (type: string)
sort order: ++
- Statistics: Num rows: 55 Data size: 5555 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 56 Data size: 5656 Basic stats: COMPLETE Column stats: COMPLETE
Execution mode: llap
LLAP IO: all inputs (cache only)
Reducer 2
@@ -547,10 +547,10 @@ STAGE PLANS:
Select Operator
expressions: KEY.reducesinkkey0 (type: tinyint), KEY.reducesinkkey1 (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 55 Data size: 5555 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 56 Data size: 5656 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
- Statistics: Num rows: 55 Data size: 5555 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 56 Data size: 5656 Basic stats: COMPLETE Column stats: COMPLETE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -596,15 +596,15 @@ STAGE PLANS:
Statistics: Num rows: 1049 Data size: 105941 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
predicate: ((not (t) IN (-1, -2, -3)) and (s like 'bob%') and s is not null and t BETWEEN 25 AND 30) (type: boolean)
- Statistics: Num rows: 55 Data size: 5555 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 56 Data size: 5656 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: t (type: tinyint), s (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 55 Data size: 5555 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 56 Data size: 5656 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col0 (type: tinyint), _col1 (type: string)
sort order: ++
- Statistics: Num rows: 55 Data size: 5555 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 56 Data size: 5656 Basic stats: COMPLETE Column stats: COMPLETE
Execution mode: llap
LLAP IO: all inputs (cache only)
Reducer 2
@@ -613,10 +613,10 @@ STAGE PLANS:
Select Operator
expressions: KEY.reducesinkkey0 (type: tinyint), KEY.reducesinkkey1 (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 55 Data size: 5555 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 56 Data size: 5656 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
- Statistics: Num rows: 55 Data size: 5555 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 56 Data size: 5656 Basic stats: COMPLETE Column stats: COMPLETE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/a1e6206c/ql/src/test/results/clientpositive/llap/subquery_multi.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/subquery_multi.q.out b/ql/src/test/results/clientpositive/llap/subquery_multi.q.out
index 988a8f4..c573b8c 100644
--- a/ql/src/test/results/clientpositive/llap/subquery_multi.q.out
+++ b/ql/src/test/results/clientpositive/llap/subquery_multi.q.out
@@ -2867,14 +2867,14 @@ STAGE PLANS:
Statistics: Num rows: 100 Data size: 10400 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
predicate: ((l_linenumber = 1) and (l_shipmode = 'AIR') and l_orderkey is not null) (type: boolean)
- Statistics: Num rows: 14 Data size: 1456 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 2 Data size: 208 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: l_orderkey (type: int), l_quantity (type: double)
outputColumnNames: _col0, _col2
- Statistics: Num rows: 14 Data size: 1442 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 2 Data size: 206 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
sort order:
- Statistics: Num rows: 14 Data size: 1442 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 2 Data size: 206 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col0 (type: int), _col2 (type: double)
Execution mode: vectorized, llap
LLAP IO: no inputs
@@ -2969,14 +2969,14 @@ STAGE PLANS:
0 _col1 (type: int), _col4 (type: int)
1 _col0 (type: int), _col1 (type: int)
outputColumnNames: _col0, _col3
- Statistics: Num rows: 7 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: _col0 (type: int), _col3 (type: int)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 7 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
- Statistics: Num rows: 7 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -2991,12 +2991,12 @@ STAGE PLANS:
0
1
outputColumnNames: _col0, _col2, _col4, _col5
- Statistics: Num rows: 14 Data size: 392 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 2 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col2 (type: double)
sort order: +
Map-reduce partition columns: _col2 (type: double)
- Statistics: Num rows: 14 Data size: 392 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 2 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col0 (type: int), _col4 (type: bigint), _col5 (type: bigint)
Reducer 8
Execution mode: llap
@@ -3008,24 +3008,24 @@ STAGE PLANS:
0 _col2 (type: double)
1 _col0 (type: double)
outputColumnNames: _col0, _col2, _col4, _col5, _col7
- Statistics: Num rows: 14 Data size: 404 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 2 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
predicate: ((_col4 = 0L) or (_col7 is null and _col2 is not null and (_col5 >= _col4))) (type: boolean)
- Statistics: Num rows: 14 Data size: 404 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 2 Data size: 64 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: _col0 (type: int), 1 (type: int)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 14 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
keys: _col0 (type: int), _col1 (type: int)
mode: hash
outputColumnNames: _col0, _col1
- Statistics: Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col0 (type: int), _col1 (type: int)
sort order: ++
Map-reduce partition columns: _col0 (type: int), _col1 (type: int)
- Statistics: Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
Stage: Stage-0
Fetch Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/a1e6206c/ql/src/test/results/clientpositive/llap/vector_decimal_cast.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/vector_decimal_cast.q.out b/ql/src/test/results/clientpositive/llap/vector_decimal_cast.q.out
index 67630b4..9e5885e 100644
--- a/ql/src/test/results/clientpositive/llap/vector_decimal_cast.q.out
+++ b/ql/src/test/results/clientpositive/llap/vector_decimal_cast.q.out
@@ -29,7 +29,7 @@ STAGE PLANS:
native: true
predicateExpression: FilterExprAndExpr(children: SelectColumnIsNotNull(col 5:double), SelectColumnIsNotNull(col 2:int), SelectColumnIsNotNull(col 10:boolean), SelectColumnIsNotNull(col 8:timestamp))
predicate: (cboolean1 is not null and cdouble is not null and cint is not null and ctimestamp1 is not null) (type: boolean)
- Statistics: Num rows: 5112 Data size: 265564 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 2945 Data size: 152996 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: cdouble (type: double), cint (type: int), cboolean1 (type: boolean), ctimestamp1 (type: timestamp), CAST( cdouble AS decimal(20,10)) (type: decimal(20,10)), CAST( cint AS decimal(23,14)) (type: decimal(23,14)), CAST( cboolean1 AS decimal(5,2)) (type: decimal(5,2)), CAST( ctimestamp1 AS decimal(15,0)) (type: decimal(15,0))
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7
@@ -38,7 +38,7 @@ STAGE PLANS:
native: true
projectedOutputColumnNums: [5, 2, 10, 8, 13, 14, 15, 16]
selectExpressions: CastDoubleToDecimal(col 5:double) -> 13:decimal(20,10), CastLongToDecimal(col 2:int) -> 14:decimal(23,14), CastLongToDecimal(col 10:boolean) -> 15:decimal(5,2), CastTimestampToDecimal(col 8:timestamp) -> 16:decimal(15,0)
- Statistics: Num rows: 5112 Data size: 2410700 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 2945 Data size: 1388804 Basic stats: COMPLETE Column stats: COMPLETE
Limit
Number of rows: 10
Limit Vectorization:
http://git-wip-us.apache.org/repos/asf/hive/blob/a1e6206c/ql/src/test/results/clientpositive/llap/vector_if_expr.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/vector_if_expr.q.out b/ql/src/test/results/clientpositive/llap/vector_if_expr.q.out
index d2edc1f..fb5cdb5 100644
--- a/ql/src/test/results/clientpositive/llap/vector_if_expr.q.out
+++ b/ql/src/test/results/clientpositive/llap/vector_if_expr.q.out
@@ -33,7 +33,7 @@ STAGE PLANS:
native: true
predicateExpression: FilterExprAndExpr(children: SelectColumnIsTrue(col 10:boolean), SelectColumnIsNotNull(col 10:boolean))
predicate: (cboolean1 and cboolean1 is not null) (type: boolean)
- Statistics: Num rows: 4587 Data size: 13704 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 3030 Data size: 9052 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: cboolean1 (type: boolean), if(cboolean1, 'first', 'second') (type: string)
outputColumnNames: _col0, _col1
@@ -42,7 +42,7 @@ STAGE PLANS:
native: true
projectedOutputColumnNums: [10, 13]
selectExpressions: IfExprStringScalarStringScalar(col 10:boolean, val first, val second) -> 13:string
- Statistics: Num rows: 4587 Data size: 857712 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 3030 Data size: 566572 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col0 (type: boolean)
sort order: +
@@ -50,7 +50,7 @@ STAGE PLANS:
className: VectorReduceSinkObjectHashOperator
native: true
nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
- Statistics: Num rows: 4587 Data size: 857712 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 3030 Data size: 566572 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col1 (type: string)
Execution mode: vectorized, llap
LLAP IO: all inputs
@@ -79,13 +79,13 @@ STAGE PLANS:
className: VectorSelectOperator
native: true
projectedOutputColumnNums: [0, 1]
- Statistics: Num rows: 4587 Data size: 857712 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 3030 Data size: 566572 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
File Sink Vectorization:
className: VectorFileSinkOperator
native: false
- Statistics: Num rows: 4587 Data size: 857712 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 3030 Data size: 566572 Basic stats: COMPLETE Column stats: COMPLETE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/a1e6206c/ql/src/test/results/clientpositive/llap/vector_mapjoin_reduce.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/vector_mapjoin_reduce.q.out b/ql/src/test/results/clientpositive/llap/vector_mapjoin_reduce.q.out
index 09a53d0..37821fb 100644
--- a/ql/src/test/results/clientpositive/llap/vector_mapjoin_reduce.q.out
+++ b/ql/src/test/results/clientpositive/llap/vector_mapjoin_reduce.q.out
@@ -403,7 +403,7 @@ STAGE PLANS:
native: true
predicateExpression: FilterExprAndExpr(children: FilterStringGroupColEqualStringScalar(col 14:string, val AIR), FilterLongColEqualLongScalar(col 3:int, val 1), SelectColumnIsNotNull(col 0:int))
predicate: ((l_linenumber = 1) and (l_shipmode = 'AIR') and l_orderkey is not null) (type: boolean)
- Statistics: Num rows: 14 Data size: 1344 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 2 Data size: 192 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: l_orderkey (type: int), 1 (type: int)
outputColumnNames: _col0, _col1
@@ -412,7 +412,7 @@ STAGE PLANS:
native: true
projectedOutputColumnNums: [0, 17]
selectExpressions: ConstantVectorExpression(val 1) -> 17:int
- Statistics: Num rows: 14 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 2 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
Group By Vectorization:
className: VectorGroupByOperator
@@ -424,7 +424,7 @@ STAGE PLANS:
keys: _col0 (type: int), _col1 (type: int)
mode: hash
outputColumnNames: _col0, _col1
- Statistics: Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col0 (type: int), _col1 (type: int)
sort order: ++
@@ -433,7 +433,7 @@ STAGE PLANS:
className: VectorReduceSinkMultiKeyOperator
native: true
nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
- Statistics: Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE
Execution mode: vectorized, llap
LLAP IO: no inputs
Map Vectorization:
@@ -494,7 +494,7 @@ STAGE PLANS:
outputColumnNames: _col0, _col3
input vertices:
1 Map 4
- Statistics: Num rows: 7 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: _col0 (type: int), _col3 (type: int)
outputColumnNames: _col0, _col1
@@ -502,13 +502,13 @@ STAGE PLANS:
className: VectorSelectOperator
native: true
projectedOutputColumnNums: [0, 2]
- Statistics: Num rows: 7 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
File Sink Vectorization:
className: VectorFileSinkOperator
native: false
- Statistics: Num rows: 7 Data size: 56 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 3 Data size: 24 Basic stats: COMPLETE Column stats: COMPLETE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/a1e6206c/ql/src/test/results/clientpositive/llap/vectorization_0.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/vectorization_0.q.out b/ql/src/test/results/clientpositive/llap/vectorization_0.q.out
index c3d810e..4206569 100644
--- a/ql/src/test/results/clientpositive/llap/vectorization_0.q.out
+++ b/ql/src/test/results/clientpositive/llap/vectorization_0.q.out
@@ -30622,17 +30622,17 @@ STAGE PLANS:
Filter Operator
isSamplingPred: false
predicate: (((cint = 45) and (cfloat = 3.02)) or ((cint = 47) and (cfloat = 2.09)) or ((cint = 49) and (cfloat = 3.5))) (type: boolean)
- Statistics: Num rows: 6 Data size: 1630 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 310 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: ctinyint (type: tinyint), csmallint (type: smallint), cint (type: int), cbigint (type: bigint), cfloat (type: float), cdouble (type: double), cstring1 (type: string), cstring2 (type: string), ctimestamp1 (type: timestamp), ctimestamp2 (type: timestamp), cboolean1 (type: boolean), cboolean2 (type: boolean)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11
- Statistics: Num rows: 6 Data size: 1630 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 310 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
GlobalTableId: 0
#### A masked pattern was here ####
NumFilesPerFileSink: 1
- Statistics: Num rows: 6 Data size: 1630 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 310 Basic stats: COMPLETE Column stats: COMPLETE
#### A masked pattern was here ####
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
@@ -30856,17 +30856,17 @@ STAGE PLANS:
Filter Operator
isSamplingPred: false
predicate: (((cint = 45) or (cfloat = 3.02)) and ((cint = 47) or (cfloat = 2.09)) and ((cint = 49) or (cfloat = 3.5))) (type: boolean)
- Statistics: Num rows: 27 Data size: 6990 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 310 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: ctinyint (type: tinyint), csmallint (type: smallint), cint (type: int), cbigint (type: bigint), cfloat (type: float), cdouble (type: double), cstring1 (type: string), cstring2 (type: string), ctimestamp1 (type: timestamp), ctimestamp2 (type: timestamp), cboolean1 (type: boolean), cboolean2 (type: boolean)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11
- Statistics: Num rows: 27 Data size: 6990 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 310 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
GlobalTableId: 0
#### A masked pattern was here ####
NumFilesPerFileSink: 1
- Statistics: Num rows: 27 Data size: 6990 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 310 Basic stats: COMPLETE Column stats: COMPLETE
#### A masked pattern was here ####
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
http://git-wip-us.apache.org/repos/asf/hive/blob/a1e6206c/ql/src/test/results/clientpositive/llap/vectorization_short_regress.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/vectorization_short_regress.q.out b/ql/src/test/results/clientpositive/llap/vectorization_short_regress.q.out
index 7362b04..a6a0371 100644
--- a/ql/src/test/results/clientpositive/llap/vectorization_short_regress.q.out
+++ b/ql/src/test/results/clientpositive/llap/vectorization_short_regress.q.out
@@ -617,7 +617,7 @@ STAGE PLANS:
native: true
predicateExpression: FilterExprOrExpr(children: FilterTimestampColEqualTimestampColumn(col 8:timestamp, col 9:timestamp), FilterDoubleColEqualDoubleScalar(col 4:float, val 762.0), FilterStringGroupColEqualStringScalar(col 6:string, val ss), FilterExprAndExpr(children: FilterLongColLessEqualLongColumn(col 1:bigint, col 3:bigint)(children: col 1:smallint), FilterLongColEqualLongScalar(col 11:boolean, val 1)), FilterExprAndExpr(children: SelectColumnIsNotNull(col 10:boolean), SelectColumnIsNotNull(col 9:timestamp), FilterStringGroupColGreaterStringScalar(col 7:string, val a)))
predicate: (((UDFToLong(csmallint) <= cbigint) and (cboolean2 = 1)) or (cboolean1 is not null and ctimestamp2 is not null and (cstring2 > 'a')) or (cfloat = 762) or (cstring1 = 'ss') or (ctimestamp1 = ctimestamp2)) (type: boolean)
- Statistics: Num rows: 12288 Data size: 3093170 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 11346 Data size: 2856120 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: cbigint (type: bigint), ctinyint (type: tinyint), csmallint (type: smallint), cint (type: int), cdouble (type: double), UDFToDouble(cbigint) (type: double), (UDFToDouble(cbigint) * UDFToDouble(cbigint)) (type: double), UDFToDouble(csmallint) (type: double), (UDFToDouble(csmallint) * UDFToDouble(csmallint)) (type: double), (cdouble * cdouble) (type: double)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9
@@ -626,7 +626,7 @@ STAGE PLANS:
native: true
projectedOutputColumnNums: [3, 0, 1, 2, 5, 13, 16, 14, 18, 15]
selectExpressions: CastLongToDouble(col 3:bigint) -> 13:double, DoubleColMultiplyDoubleColumn(col 14:double, col 15:double)(children: CastLongToDouble(col 3:bigint) -> 14:double, CastLongToDouble(col 3:bigint) -> 15:double) -> 16:double, CastLongToDouble(col 1:smallint) -> 14:double, DoubleColMultiplyDoubleColumn(col 15:double, col 17:double)(children: CastLongToDouble(col 1:smallint) -> 15:double, CastLongToDouble(col 1:smallint) -> 17:double) -> 18:double, DoubleColMultiplyDoubleColumn(col 5:double, col 5:double) -> 15:double
- Statistics: Num rows: 12288 Data size: 3093170 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 11346 Data size: 2856120 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
aggregations: sum(_col6), sum(_col5), count(_col0), count(), max(_col1), sum(_col8), sum(_col7), count(_col2), max(_col3), sum(_col9), sum(_col4), count(_col4), count(_col1), sum(_col1)
Group By Vectorization:
@@ -3184,7 +3184,7 @@ STAGE PLANS:
native: true
predicateExpression: FilterExprAndExpr(children: FilterExprOrExpr(children: FilterExprAndExpr(children: FilterDoubleColLessDoubleColumn(col 5:double, col 13:double)(children: CastLongToDouble(col 1:smallint) -> 13:double), FilterLongColEqualLongColumn(col 11:boolean, col 10:boolean), FilterDecimalColLessEqualDecimalScalar(col 14:decimal(22,3), val -863.257)(children: CastLongToDecimal(col 3:bigint) -> 14:decimal(22,3))), FilterExprAndExpr(children: FilterLongColGreaterEqualLongScalar(col 2:int, val -257), SelectColumnIsNotNull(col 6:string), FilterLongColGreaterEqualLongScalar(col 10:boolean, val 1)), FilterStringColRegExpStringScalar(col 7:string, pattern b), FilterExprAndExpr(children: FilterLongColGreaterEqualLongColumn(col 1:smallint, col 0:smallint)(children: col 0:tinyint), SelectColumnIsNull(col 9:timestamp))), SelectColumnIsNotNull(col 10:boolean))
predicate: ((((cdouble < UDFToDouble(csmallint)) and (cboolean2 = cboolean1) and (CAST( cbigint AS decimal(22,3)) <= -863.257)) or ((cint >= -257) and cstring1 is not null and (cboolean1 >= 1)) or cstring2 regexp 'b' or ((csmallint >= UDFToShort(ctinyint)) and ctimestamp2 is null)) and cboolean1 is not null) (type: boolean)
- Statistics: Num rows: 7845 Data size: 1661020 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 7153 Data size: 1514550 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: cboolean1 (type: boolean), cfloat (type: float), cbigint (type: bigint), cint (type: int), cdouble (type: double), ctinyint (type: tinyint), csmallint (type: smallint), UDFToDouble(cint) (type: double), (UDFToDouble(cint) * UDFToDouble(cint)) (type: double), UDFToDouble(cbigint) (type: double), (UDFToDouble(cbigint) * UDFToDouble(cbigint)) (type: double), UDFToDouble(ctinyint) (type: double), (UDFToDouble(ctinyint) * UDFToDouble(ctinyint)) (type: double), UDFToDouble(csmallint) (type: double), (UDFToDouble(csmallint) * UDFToDouble(csmallint)) (type: double)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14
@@ -3193,7 +3193,7 @@ STAGE PLANS:
native: true
projectedOutputColumnNums: [10, 4, 3, 2, 5, 0, 1, 13, 17, 15, 19, 16, 21, 18, 23]
selectExpressions: CastLongToDouble(col 2:int) -> 13:double, DoubleColMultiplyDoubleColumn(col 15:double, col 16:double)(children: CastLongToDouble(col 2:int) -> 15:double, CastLongToDouble(col 2:int) -> 16:double) -> 17:double, CastLongToDouble(col 3:bigint) -> 15:double, DoubleColMultiplyDoubleColumn(col 16:double, col 18:double)(children: CastLongToDouble(col 3:bigint) -> 16:double, CastLongToDouble(col 3:bigint) -> 18:double) -> 19:double, CastLongToDouble(col 0:tinyint) -> 16:double, DoubleColMultiplyDoubleColumn(col 18:double, col 20:double)(children: CastLongToDouble(col 0:tinyint) -> 18:double, CastLongToDouble(col 0:tinyint) -> 20:double) -> 21:double, CastLongToDouble(col 1:smallint) -> 18:double, DoubleColMultiplyDoubleColumn(col 20:double, col 22:double)(children: CastLongToDouble(col 1:smallint) -> 20:double, CastLongToDouble(col 1:smallint) -> 22:double) -> 23:double
- Statistics: Num rows: 7845 Data size: 1661020 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 7153 Data size: 1514550 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
aggregations: max(_col1), sum(_col2), sum(_col8), sum(_col7), count(_col3), sum(_col4), count(_col4), min(_col2), sum(_col10), sum(_col9), count(_col2), sum(_col3), sum(_col12), sum(_col11), count(_col5), sum(_col14), sum(_col13), count(_col6)
Group By Vectorization:
http://git-wip-us.apache.org/repos/asf/hive/blob/a1e6206c/ql/src/test/results/clientpositive/llap/vectorized_nested_mapjoin.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/vectorized_nested_mapjoin.q.out b/ql/src/test/results/clientpositive/llap/vectorized_nested_mapjoin.q.out
index a2ca7f8..80e1cab 100644
--- a/ql/src/test/results/clientpositive/llap/vectorized_nested_mapjoin.q.out
+++ b/ql/src/test/results/clientpositive/llap/vectorized_nested_mapjoin.q.out
@@ -40,11 +40,11 @@ STAGE PLANS:
outputColumnNames: _col2, _col3
input vertices:
1 Map 3
- Statistics: Num rows: 661228 Data size: 7913928 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 585044 Data size: 7002120 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: _col2 (type: smallint), _col3 (type: double)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 661228 Data size: 7913928 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 585044 Data size: 7002120 Basic stats: COMPLETE Column stats: COMPLETE
Map Join Operator
condition map:
Inner Join 0 to 1
@@ -54,7 +54,7 @@ STAGE PLANS:
outputColumnNames: _col1
input vertices:
1 Map 4
- Statistics: Num rows: 1452263 Data size: 11604232 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1284939 Data size: 10267240 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
aggregations: sum(_col1)
mode: hash
@@ -82,16 +82,16 @@ STAGE PLANS:
Statistics: Num rows: 12288 Data size: 146796 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
predicate: (csmallint is not null and ctinyint is not null) (type: boolean)
- Statistics: Num rows: 6848 Data size: 81820 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 6059 Data size: 72396 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: ctinyint (type: tinyint), csmallint (type: smallint), cdouble (type: double)
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 6848 Data size: 81820 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 6059 Data size: 72396 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col0 (type: tinyint)
sort order: +
Map-reduce partition columns: _col0 (type: tinyint)
- Statistics: Num rows: 6848 Data size: 81820 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 6059 Data size: 72396 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col1 (type: smallint), _col2 (type: double)
Execution mode: vectorized, llap
LLAP IO: all inputs
http://git-wip-us.apache.org/repos/asf/hive/blob/a1e6206c/ql/src/test/results/clientpositive/spark/spark_explainuser_1.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/spark_explainuser_1.q.out b/ql/src/test/results/clientpositive/spark/spark_explainuser_1.q.out
index bdf9857..a47f51d 100644
--- a/ql/src/test/results/clientpositive/spark/spark_explainuser_1.q.out
+++ b/ql/src/test/results/clientpositive/spark/spark_explainuser_1.q.out
@@ -387,25 +387,25 @@ Stage-0
<-Map 5 [GROUP]
GROUP [RS_7]
PartitionCols:_col0, _col1, _col2
- Group By Operator [GBY_6] (rows=2 width=101)
+ Group By Operator [GBY_6] (rows=1 width=101)
Output:["_col0","_col1","_col2","_col3"],aggregations:["sum(c_int)"],keys:key, c_int, c_float
- Filter Operator [FIL_33] (rows=5 width=93)
+ Filter Operator [FIL_33] (rows=4 width=93)
predicate:(((c_int + 1) >= 0) and ((c_int > 0) or (c_float >= 0)) and key is not null)
TableScan [TS_3] (rows=20 width=88)
default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"]
<-Reducer 8 [PARTITION-LEVEL SORT]
PARTITION-LEVEL SORT [RS_19]
PartitionCols:_col0
- Select Operator [SEL_16] (rows=2 width=89)
+ Select Operator [SEL_16] (rows=1 width=89)
Output:["_col0","_col1"]
- Group By Operator [GBY_15] (rows=2 width=93)
+ Group By Operator [GBY_15] (rows=1 width=93)
Output:["_col0","_col1","_col2"],keys:KEY._col0, KEY._col1, KEY._col2
<-Map 7 [GROUP]
GROUP [RS_14]
PartitionCols:_col0, _col1, _col2
- Group By Operator [GBY_13] (rows=2 width=93)
+ Group By Operator [GBY_13] (rows=1 width=93)
Output:["_col0","_col1","_col2"],keys:key, c_int, c_float
- Filter Operator [FIL_34] (rows=5 width=93)
+ Filter Operator [FIL_34] (rows=4 width=93)
predicate:(((c_int + 1) >= 0) and ((c_int > 0) or (c_float >= 0)) and key is not null)
TableScan [TS_10] (rows=20 width=88)
default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"]
@@ -1093,27 +1093,27 @@ Stage-0
Stage-1
Reducer 2
File Output Operator [FS_16]
- Select Operator [SEL_15] (rows=40 width=101)
+ Select Operator [SEL_15] (rows=48 width=101)
Output:["_col0","_col1","_col2","_col3","_col4"]
- Filter Operator [FIL_13] (rows=40 width=101)
+ Filter Operator [FIL_13] (rows=48 width=101)
predicate:(((_col1 + _col4) = 2) and ((_col1 > 0) or (_col6 >= 0)))
- Join Operator [JOIN_12] (rows=81 width=100)
+ Join Operator [JOIN_12] (rows=96 width=100)
Output:["_col1","_col2","_col3","_col4","_col6"],condition map:[{"":"{\"type\":\"Inner\",\"left\":0,\"right\":1}"},{"":"{\"type\":\"Inner\",\"left\":0,\"right\":2}"}],keys:{"0":"_col0","1":"_col0","2":"_col0"}
<-Map 1 [PARTITION-LEVEL SORT]
PARTITION-LEVEL SORT [RS_9]
PartitionCols:_col0
- Select Operator [SEL_2] (rows=9 width=93)
+ Select Operator [SEL_2] (rows=8 width=93)
Output:["_col0","_col1","_col2"]
- Filter Operator [FIL_17] (rows=9 width=93)
+ Filter Operator [FIL_17] (rows=8 width=93)
predicate:(((c_int + 1) = 2) and ((c_int > 0) or (c_float >= 0)) and key is not null)
TableScan [TS_0] (rows=20 width=88)
default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"]
<-Map 3 [PARTITION-LEVEL SORT]
PARTITION-LEVEL SORT [RS_10]
PartitionCols:_col0
- Select Operator [SEL_5] (rows=9 width=89)
+ Select Operator [SEL_5] (rows=8 width=89)
Output:["_col0","_col1"]
- Filter Operator [FIL_18] (rows=9 width=93)
+ Filter Operator [FIL_18] (rows=8 width=93)
predicate:(((c_int + 1) = 2) and ((c_int > 0) or (c_float >= 0)) and key is not null)
TableScan [TS_3] (rows=20 width=88)
default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"]
@@ -1142,27 +1142,27 @@ Stage-0
Stage-1
Reducer 2
File Output Operator [FS_16]
- Select Operator [SEL_15] (rows=40 width=101)
+ Select Operator [SEL_15] (rows=48 width=101)
Output:["_col0","_col1","_col2","_col3","_col4"]
- Filter Operator [FIL_13] (rows=40 width=101)
+ Filter Operator [FIL_13] (rows=48 width=101)
predicate:(((_col1 + _col4) = 2) and ((_col1 > 0) or (_col6 >= 0)))
- Join Operator [JOIN_12] (rows=81 width=100)
+ Join Operator [JOIN_12] (rows=96 width=100)
Output:["_col1","_col2","_col3","_col4","_col6"],condition map:[{"":"{\"type\":\"Inner\",\"left\":0,\"right\":1}"},{"":"{\"type\":\"Inner\",\"left\":0,\"right\":2}"}],keys:{"0":"_col0","1":"_col0","2":"_col0"}
<-Map 1 [PARTITION-LEVEL SORT]
PARTITION-LEVEL SORT [RS_9]
PartitionCols:_col0
- Select Operator [SEL_2] (rows=9 width=93)
+ Select Operator [SEL_2] (rows=8 width=93)
Output:["_col0","_col1","_col2"]
- Filter Operator [FIL_17] (rows=9 width=93)
+ Filter Operator [FIL_17] (rows=8 width=93)
predicate:(((c_int + 1) = 2) and ((c_int > 0) or (c_float >= 0)) and key is not null)
TableScan [TS_0] (rows=20 width=88)
default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"]
<-Map 3 [PARTITION-LEVEL SORT]
PARTITION-LEVEL SORT [RS_10]
PartitionCols:_col0
- Select Operator [SEL_5] (rows=9 width=89)
+ Select Operator [SEL_5] (rows=8 width=89)
Output:["_col0","_col1"]
- Filter Operator [FIL_18] (rows=9 width=93)
+ Filter Operator [FIL_18] (rows=8 width=93)
predicate:(((c_int + 1) = 2) and ((c_int > 0) or (c_float >= 0)) and key is not null)
TableScan [TS_3] (rows=20 width=88)
default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"]
@@ -1427,16 +1427,16 @@ Stage-0
Stage-1
Reducer 2
File Output Operator [FS_12]
- Select Operator [SEL_11] (rows=9 width=4)
+ Select Operator [SEL_11] (rows=8 width=4)
Output:["_col0"]
- Join Operator [JOIN_10] (rows=9 width=4)
+ Join Operator [JOIN_10] (rows=8 width=4)
Output:["_col1"],condition map:[{"":"{\"type\":\"Left Semi\",\"left\":0,\"right\":1}"}],keys:{"0":"_col0","1":"_col0"}
<-Map 1 [PARTITION-LEVEL SORT]
PARTITION-LEVEL SORT [RS_8]
PartitionCols:_col0
- Select Operator [SEL_2] (rows=9 width=93)
+ Select Operator [SEL_2] (rows=8 width=93)
Output:["_col0","_col1"]
- Filter Operator [FIL_13] (rows=9 width=93)
+ Filter Operator [FIL_13] (rows=8 width=93)
predicate:(((c_int + 1) = 2) and ((c_int > 0) or (c_float >= 0)) and key is not null)
TableScan [TS_0] (rows=20 width=88)
default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"]
@@ -1474,20 +1474,20 @@ Stage-0
<-Map 1 [PARTITION-LEVEL SORT]
PARTITION-LEVEL SORT [RS_13]
PartitionCols:_col0
- Select Operator [SEL_2] (rows=9 width=93)
+ Select Operator [SEL_2] (rows=8 width=93)
Output:["_col0","_col1","_col2"]
- Filter Operator [FIL_19] (rows=9 width=93)
+ Filter Operator [FIL_19] (rows=8 width=93)
predicate:(((c_int + 1) = 2) and ((c_int > 0) or (c_float >= 0)) and key is not null)
TableScan [TS_0] (rows=20 width=88)
default@cbo_t1,cbo_t1,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"]
<-Map 3 [PARTITION-LEVEL SORT]
PARTITION-LEVEL SORT [RS_14]
PartitionCols:_col0
- Group By Operator [GBY_10] (rows=4 width=85)
+ Group By Operator [GBY_10] (rows=3 width=85)
Output:["_col0"],keys:_col0
- Select Operator [SEL_5] (rows=9 width=85)
+ Select Operator [SEL_5] (rows=8 width=85)
Output:["_col0"]
- Filter Operator [FIL_20] (rows=9 width=93)
+ Filter Operator [FIL_20] (rows=8 width=93)
predicate:(((c_int + 1) = 2) and ((c_int > 0) or (c_float >= 0)) and key is not null)
TableScan [TS_3] (rows=20 width=88)
default@cbo_t2,cbo_t2,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int","c_float"]
@@ -1974,18 +1974,18 @@ Stage-0
Stage-1
Reducer 4
File Output Operator [FS_22]
- Select Operator [SEL_21] (rows=7 width=8)
+ Select Operator [SEL_21] (rows=3 width=8)
Output:["_col0","_col1"]
- Join Operator [JOIN_20] (rows=7 width=8)
+ Join Operator [JOIN_20] (rows=3 width=8)
Output:["_col0","_col3"],condition map:[{"":"{\"type\":\"Left Semi\",\"left\":0,\"right\":1}"}],keys:{"0":"_col1, _col4","1":"_col0, _col1"}
<-Map 6 [PARTITION-LEVEL SORT]
PARTITION-LEVEL SORT [RS_19]
PartitionCols:_col0, _col1
- Group By Operator [GBY_17] (rows=4 width=8)
+ Group By Operator [GBY_17] (rows=1 width=8)
Output:["_col0","_col1"],keys:_col0, _col1
- Select Operator [SEL_12] (rows=14 width=8)
+ Select Operator [SEL_12] (rows=2 width=8)
Output:["_col0","_col1"]
- Filter Operator [FIL_25] (rows=14 width=96)
+ Filter Operator [FIL_25] (rows=2 width=96)
predicate:((l_linenumber = 1) and (l_shipmode = 'AIR') and l_orderkey is not null)
TableScan [TS_10] (rows=100 width=96)
default@lineitem,lineitem,Tbl:COMPLETE,Col:COMPLETE,Output:["l_orderkey","l_linenumber","l_shipmode"]
[2/4] hive git commit: Revert "HIVE-19557: stats: filters for dates
are not taking advantage of min/max values (Zoltan Haindrich reviewed by
Ashutosh Chauhan)"
Posted by jc...@apache.org.
Revert "HIVE-19557: stats: filters for dates are not taking advantage of min/max values (Zoltan Haindrich reviewed by Ashutosh Chauhan)"
This reverts commit 0b2d364aa5977c7a15a4b5082cf9ef431dcc394e.
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/7056445c
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/7056445c
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/7056445c
Branch: refs/heads/master
Commit: 7056445c1e9a1b040ae44dca99b351296325b047
Parents: 29f5784
Author: Jesus Camacho Rodriguez <jc...@apache.org>
Authored: Wed May 23 09:51:40 2018 -0700
Committer: Jesus Camacho Rodriguez <jc...@apache.org>
Committed: Wed May 23 09:51:40 2018 -0700
----------------------------------------------------------------------
.../test/resources/testconfiguration.properties | 1 -
.../stats/annotation/StatsRulesProcFactory.java | 10 +-
.../clientpositive/colstats_date_min_max.q | 30 ---
.../llap/colstats_date_min_max.q.out | 193 -------------------
.../clientpositive/llap/vector_between_in.q.out | 20 +-
5 files changed, 11 insertions(+), 243 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/7056445c/itests/src/test/resources/testconfiguration.properties
----------------------------------------------------------------------
diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties
index 6007d5a..6528ec6 100644
--- a/itests/src/test/resources/testconfiguration.properties
+++ b/itests/src/test/resources/testconfiguration.properties
@@ -488,7 +488,6 @@ minillaplocal.query.files=\
cbo_subq_not_in.q,\
column_table_stats.q,\
column_table_stats_orc.q,\
- colstats_date_min_max.q,\
compare_double_bigint_2.q,\
constprog_dpp.q,\
current_date_timestamp.q,\
http://git-wip-us.apache.org/repos/asf/hive/blob/7056445c/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java
index 91cccfb..c770227 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java
@@ -100,8 +100,6 @@ import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPNull;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPOr;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFStruct;
import org.apache.hadoop.hive.serde.serdeConstants;
-import org.apache.hadoop.hive.serde2.io.DateWritable;
-import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
@@ -753,14 +751,8 @@ public class StatsRulesProcFactory {
}
} else if (colTypeLowerCase.equals(serdeConstants.INT_TYPE_NAME) ||
colTypeLowerCase.equals(serdeConstants.DATE_TYPE_NAME)) {
- int value;
- if (colTypeLowerCase == serdeConstants.DATE_TYPE_NAME) {
- DateWritable writableVal = new DateWritable(java.sql.Date.valueOf(boundValue));
- value = writableVal.getDays();
- } else {
- value = new Integer(boundValue);
- }
// Date is an integer internally
+ int value = new Integer(boundValue);
int maxValue = cs.getRange().maxValue.intValue();
int minValue = cs.getRange().minValue.intValue();
if (upperBound) {
http://git-wip-us.apache.org/repos/asf/hive/blob/7056445c/ql/src/test/queries/clientpositive/colstats_date_min_max.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/colstats_date_min_max.q b/ql/src/test/queries/clientpositive/colstats_date_min_max.q
deleted file mode 100644
index 7f5be6a..0000000
--- a/ql/src/test/queries/clientpositive/colstats_date_min_max.q
+++ /dev/null
@@ -1,30 +0,0 @@
-set hive.explain.user=true;
-
-create table d1(d date);
--- tblproperties('transactional'='false');
-
-insert into d1 values
- ('2010-10-01'),
- ('2010-10-02'),
- ('2010-10-03'),
- ('2010-10-04'),
- ('2010-10-05'),
- ('2010-10-06'),
- ('2010-10-07'),
- ('2010-10-08'),
- ('2010-10-09'),
- ('2010-10-10');
-
-analyze table d1 compute statistics for columns;
-
-desc formatted d1;
-desc formatted d1 d;
-
-explain
-select 'stats: FIL ~0 read',count(1) from d1 where d < '2010-03-01';
-
-explain
-select 'stats: FIL estimate some read',count(1) from d1 where d < '2010-10-03';
-
-explain
-select 'stats: FIL estimate all read',count(1) from d1 where d < '2010-11-03';
http://git-wip-us.apache.org/repos/asf/hive/blob/7056445c/ql/src/test/results/clientpositive/llap/colstats_date_min_max.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/colstats_date_min_max.q.out b/ql/src/test/results/clientpositive/llap/colstats_date_min_max.q.out
deleted file mode 100644
index 7754f3e..0000000
--- a/ql/src/test/results/clientpositive/llap/colstats_date_min_max.q.out
+++ /dev/null
@@ -1,193 +0,0 @@
-PREHOOK: query: create table d1(d date)
-PREHOOK: type: CREATETABLE
-PREHOOK: Output: database:default
-PREHOOK: Output: default@d1
-POSTHOOK: query: create table d1(d date)
-POSTHOOK: type: CREATETABLE
-POSTHOOK: Output: database:default
-POSTHOOK: Output: default@d1
-PREHOOK: query: insert into d1 values
- ('2010-10-01'),
- ('2010-10-02'),
- ('2010-10-03'),
- ('2010-10-04'),
- ('2010-10-05'),
- ('2010-10-06'),
- ('2010-10-07'),
- ('2010-10-08'),
- ('2010-10-09'),
- ('2010-10-10')
-PREHOOK: type: QUERY
-PREHOOK: Input: _dummy_database@_dummy_table
-PREHOOK: Output: default@d1
-POSTHOOK: query: insert into d1 values
- ('2010-10-01'),
- ('2010-10-02'),
- ('2010-10-03'),
- ('2010-10-04'),
- ('2010-10-05'),
- ('2010-10-06'),
- ('2010-10-07'),
- ('2010-10-08'),
- ('2010-10-09'),
- ('2010-10-10')
-POSTHOOK: type: QUERY
-POSTHOOK: Input: _dummy_database@_dummy_table
-POSTHOOK: Output: default@d1
-POSTHOOK: Lineage: d1.d SCRIPT []
-PREHOOK: query: analyze table d1 compute statistics for columns
-PREHOOK: type: ANALYZE_TABLE
-PREHOOK: Input: default@d1
-PREHOOK: Output: default@d1
-#### A masked pattern was here ####
-POSTHOOK: query: analyze table d1 compute statistics for columns
-POSTHOOK: type: ANALYZE_TABLE
-POSTHOOK: Input: default@d1
-POSTHOOK: Output: default@d1
-#### A masked pattern was here ####
-PREHOOK: query: desc formatted d1
-PREHOOK: type: DESCTABLE
-PREHOOK: Input: default@d1
-POSTHOOK: query: desc formatted d1
-POSTHOOK: type: DESCTABLE
-POSTHOOK: Input: default@d1
-# col_name data_type comment
-d date
-
-# Detailed Table Information
-Database: default
-#### A masked pattern was here ####
-Retention: 0
-#### A masked pattern was here ####
-Table Type: MANAGED_TABLE
-Table Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"d\":\"true\"}}
- bucketing_version 2
- numFiles 1
- numRows 10
- rawDataSize 100
- totalSize 110
-#### A masked pattern was here ####
-
-# Storage Information
-SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-InputFormat: org.apache.hadoop.mapred.TextInputFormat
-OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
-Compressed: No
-Num Buckets: -1
-Bucket Columns: []
-Sort Columns: []
-Storage Desc Params:
- serialization.format 1
-PREHOOK: query: desc formatted d1 d
-PREHOOK: type: DESCTABLE
-PREHOOK: Input: default@d1
-POSTHOOK: query: desc formatted d1 d
-POSTHOOK: type: DESCTABLE
-POSTHOOK: Input: default@d1
-col_name d
-data_type date
-min 2010-10-01
-max 2010-10-10
-num_nulls 0
-distinct_count 10
-avg_col_len
-max_col_len
-num_trues
-num_falses
-bitVector HL
-comment from deserializer
-COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"d\":\"true\"}}
-PREHOOK: query: explain
-select 'stats: FIL ~0 read',count(1) from d1 where d < '2010-03-01'
-PREHOOK: type: QUERY
-POSTHOOK: query: explain
-select 'stats: FIL ~0 read',count(1) from d1 where d < '2010-03-01'
-POSTHOOK: type: QUERY
-Plan optimized by CBO.
-
-Vertex dependency in root stage
-Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE)
-
-Stage-0
- Fetch Operator
- limit:-1
- Stage-1
- Reducer 2 vectorized, llap
- File Output Operator [FS_15]
- Select Operator [SEL_14] (rows=1 width=110)
- Output:["_col0","_col1"]
- Group By Operator [GBY_13] (rows=1 width=8)
- Output:["_col0"],aggregations:["count(VALUE._col0)"]
- <-Map 1 [CUSTOM_SIMPLE_EDGE] vectorized, llap
- PARTITION_ONLY_SHUFFLE [RS_12]
- Group By Operator [GBY_11] (rows=1 width=8)
- Output:["_col0"],aggregations:["count()"]
- Select Operator [SEL_10] (rows=1 width=56)
- Filter Operator [FIL_9] (rows=1 width=56)
- predicate:(d < DATE'2010-03-01')
- TableScan [TS_0] (rows=10 width=56)
- default@d1,d1,Tbl:COMPLETE,Col:COMPLETE,Output:["d"]
-
-PREHOOK: query: explain
-select 'stats: FIL estimate some read',count(1) from d1 where d < '2010-10-03'
-PREHOOK: type: QUERY
-POSTHOOK: query: explain
-select 'stats: FIL estimate some read',count(1) from d1 where d < '2010-10-03'
-POSTHOOK: type: QUERY
-Plan optimized by CBO.
-
-Vertex dependency in root stage
-Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE)
-
-Stage-0
- Fetch Operator
- limit:-1
- Stage-1
- Reducer 2 vectorized, llap
- File Output Operator [FS_15]
- Select Operator [SEL_14] (rows=1 width=121)
- Output:["_col0","_col1"]
- Group By Operator [GBY_13] (rows=1 width=8)
- Output:["_col0"],aggregations:["count(VALUE._col0)"]
- <-Map 1 [CUSTOM_SIMPLE_EDGE] vectorized, llap
- PARTITION_ONLY_SHUFFLE [RS_12]
- Group By Operator [GBY_11] (rows=1 width=8)
- Output:["_col0"],aggregations:["count()"]
- Select Operator [SEL_10] (rows=3 width=56)
- Filter Operator [FIL_9] (rows=3 width=56)
- predicate:(d < DATE'2010-10-03')
- TableScan [TS_0] (rows=10 width=56)
- default@d1,d1,Tbl:COMPLETE,Col:COMPLETE,Output:["d"]
-
-PREHOOK: query: explain
-select 'stats: FIL estimate all read',count(1) from d1 where d < '2010-11-03'
-PREHOOK: type: QUERY
-POSTHOOK: query: explain
-select 'stats: FIL estimate all read',count(1) from d1 where d < '2010-11-03'
-POSTHOOK: type: QUERY
-Plan optimized by CBO.
-
-Vertex dependency in root stage
-Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE)
-
-Stage-0
- Fetch Operator
- limit:-1
- Stage-1
- Reducer 2 vectorized, llap
- File Output Operator [FS_15]
- Select Operator [SEL_14] (rows=1 width=120)
- Output:["_col0","_col1"]
- Group By Operator [GBY_13] (rows=1 width=8)
- Output:["_col0"],aggregations:["count(VALUE._col0)"]
- <-Map 1 [CUSTOM_SIMPLE_EDGE] vectorized, llap
- PARTITION_ONLY_SHUFFLE [RS_12]
- Group By Operator [GBY_11] (rows=1 width=8)
- Output:["_col0"],aggregations:["count()"]
- Select Operator [SEL_10] (rows=10 width=56)
- Filter Operator [FIL_9] (rows=10 width=56)
- predicate:(d < DATE'2010-11-03')
- TableScan [TS_0] (rows=10 width=56)
- default@d1,d1,Tbl:COMPLETE,Col:COMPLETE,Output:["d"]
-
http://git-wip-us.apache.org/repos/asf/hive/blob/7056445c/ql/src/test/results/clientpositive/llap/vector_between_in.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/vector_between_in.q.out b/ql/src/test/results/clientpositive/llap/vector_between_in.q.out
index f76053e..6093beb 100644
--- a/ql/src/test/results/clientpositive/llap/vector_between_in.q.out
+++ b/ql/src/test/results/clientpositive/llap/vector_between_in.q.out
@@ -465,7 +465,7 @@ STAGE PLANS:
native: true
predicateExpression: FilterLongColumnBetween(col 3:date, left -2, right 1)
predicate: cdate BETWEEN DATE'1969-12-30' AND DATE'1970-01-02' (type: boolean)
- Statistics: Num rows: 4096 Data size: 217934 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1365 Data size: 72627 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: cdate (type: date)
outputColumnNames: _col0
@@ -473,7 +473,7 @@ STAGE PLANS:
className: VectorSelectOperator
native: true
projectedOutputColumnNums: [3]
- Statistics: Num rows: 4096 Data size: 217934 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1365 Data size: 72627 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: date)
sort order: +
@@ -481,7 +481,7 @@ STAGE PLANS:
className: VectorReduceSinkObjectHashOperator
native: true
nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
- Statistics: Num rows: 4096 Data size: 217934 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1365 Data size: 72627 Basic stats: COMPLETE Column stats: NONE
Execution mode: vectorized, llap
LLAP IO: all inputs
Map Vectorization:
@@ -509,13 +509,13 @@ STAGE PLANS:
className: VectorSelectOperator
native: true
projectedOutputColumnNums: [0]
- Statistics: Num rows: 4096 Data size: 217934 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1365 Data size: 72627 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
File Sink Vectorization:
className: VectorFileSinkOperator
native: false
- Statistics: Num rows: 4096 Data size: 217934 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1365 Data size: 72627 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -560,7 +560,7 @@ STAGE PLANS:
native: true
predicateExpression: FilterLongColumnNotBetween(col 3:date, left -610, right 608)
predicate: cdate NOT BETWEEN DATE'1968-05-01' AND DATE'1971-09-01' (type: boolean)
- Statistics: Num rows: 8193 Data size: 435921 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 10924 Data size: 581228 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: cdate (type: date)
outputColumnNames: _col0
@@ -568,7 +568,7 @@ STAGE PLANS:
className: VectorSelectOperator
native: true
projectedOutputColumnNums: [3]
- Statistics: Num rows: 8193 Data size: 435921 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 10924 Data size: 581228 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: date)
sort order: +
@@ -576,7 +576,7 @@ STAGE PLANS:
className: VectorReduceSinkObjectHashOperator
native: true
nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
- Statistics: Num rows: 8193 Data size: 435921 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 10924 Data size: 581228 Basic stats: COMPLETE Column stats: NONE
Execution mode: vectorized, llap
LLAP IO: all inputs
Map Vectorization:
@@ -604,13 +604,13 @@ STAGE PLANS:
className: VectorSelectOperator
native: true
projectedOutputColumnNums: [0]
- Statistics: Num rows: 8193 Data size: 435921 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 10924 Data size: 581228 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
File Sink Vectorization:
className: VectorFileSinkOperator
native: false
- Statistics: Num rows: 8193 Data size: 435921 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 10924 Data size: 581228 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat