You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by de...@apache.org on 2023/05/12 03:34:37 UTC
[hive] branch master updated: HIVE-27163: Column stats are not getting published after an insert into an external table with custom location (Zhihua Deng, reviewed by Krisztian Kasa)
This is an automated email from the ASF dual-hosted git repository.
dengzh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new dd8a867386b HIVE-27163: Column stats are not getting published after an insert into an external table with custom location (Zhihua Deng, reviewed by Krisztian Kasa)
dd8a867386b is described below
commit dd8a867386b605ecd7e8dbec476556bf24f54c6f
Author: dengzh <de...@gmail.com>
AuthorDate: Fri May 12 11:34:25 2023 +0800
HIVE-27163: Column stats are not getting published after an insert into an external table with custom location (Zhihua Deng, reviewed by Krisztian Kasa)
Closes #4228
---
.../src/test/results/positive/col_stats.q.out | 19 ++++----
.../results/positive/truncate_iceberg_table.q.out | 2 +-
.../hive/ql/ddl/table/create/CreateTableDesc.java | 29 ++++++++----
.../clientpositive/stats_external_location.q | 9 ++++
.../clientpositive/llap/default_file_format.q.out | 14 ++++++
.../test/results/clientpositive/llap/mm_exim.q.out | 4 +-
.../llap/stats_external_location.q.out | 47 ++++++++++++++++++++
.../llap/translated_external_rename3.q.out | 3 --
.../apache/hadoop/hive/common/StatsSetupConst.java | 51 ++++++++++++++++++++++
.../apache/hadoop/hive/metastore/Warehouse.java | 16 +++++--
.../apache/hadoop/hive/metastore/HMSHandler.java | 6 +--
.../hive/metastore/utils/MetaStoreServerUtils.java | 51 +++++++++++++++++++++-
12 files changed, 218 insertions(+), 33 deletions(-)
diff --git a/iceberg/iceberg-handler/src/test/results/positive/col_stats.q.out b/iceberg/iceberg-handler/src/test/results/positive/col_stats.q.out
index b1f13fa76b5..851cb106ac3 100644
--- a/iceberg/iceberg-handler/src/test/results/positive/col_stats.q.out
+++ b/iceberg/iceberg-handler/src/test/results/positive/col_stats.q.out
@@ -244,14 +244,14 @@ Stage-0
Stage-1
Reducer 2 vectorized
File Output Operator [FS_8]
- Select Operator [SEL_7] (rows=9 width=95)
+ Select Operator [SEL_7] (rows=9 width=192)
Output:["_col0","_col1","_col2"]
<-Map 1 [SIMPLE_EDGE] vectorized
SHUFFLE [RS_6]
- Select Operator [SEL_5] (rows=9 width=95)
+ Select Operator [SEL_5] (rows=9 width=192)
Output:["_col0","_col1","_col2"]
- TableScan [TS_0] (rows=9 width=95)
- default@tbl_ice_puffin,tbl_ice_puffin,Tbl:COMPLETE,Col:COMPLETE,Output:["a","b","c"]
+ TableScan [TS_0] (rows=9 width=192)
+ default@tbl_ice_puffin,tbl_ice_puffin,Tbl:COMPLETE,Col:NONE,Output:["a","b","c"]
PREHOOK: query: drop table if exists tbl_ice_puffin
PREHOOK: type: DROPTABLE
@@ -339,17 +339,16 @@ POSTHOOK: type: DESCTABLE
POSTHOOK: Input: default@tbl_ice_puffin
col_name a
data_type int
-min 1
-max 333
-num_nulls 0
-distinct_count 7
+min
+max
+num_nulls
+distinct_count
avg_col_len
max_col_len
num_trues
num_falses
-bit_vector HL
+bit_vector
comment
-COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"a\":\"true\",\"b\":\"true\",\"c\":\"true\"}}
PREHOOK: query: drop table if exists tbl_ice
PREHOOK: type: DROPTABLE
POSTHOOK: query: drop table if exists tbl_ice
diff --git a/iceberg/iceberg-handler/src/test/results/positive/truncate_iceberg_table.q.out b/iceberg/iceberg-handler/src/test/results/positive/truncate_iceberg_table.q.out
index dfab2edec75..07e2a34e423 100644
--- a/iceberg/iceberg-handler/src/test/results/positive/truncate_iceberg_table.q.out
+++ b/iceberg/iceberg-handler/src/test/results/positive/truncate_iceberg_table.q.out
@@ -225,7 +225,7 @@ Retention: 0
#### A masked pattern was here ####
Table Type: EXTERNAL_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"id\":\"true\",\"value\":\"true\"}}
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
EXTERNAL TRUE
bucketing_version 2
current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"id\",\"required\":false,\"type\":\"int\"},{\"id\":2,\"name\":\"value\",\"required\":false,\"type\":\"string\"}]}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/create/CreateTableDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/create/CreateTableDesc.java
index a228cca5045..8e2ca07b384 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/create/CreateTableDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/create/CreateTableDesc.java
@@ -20,10 +20,13 @@ package org.apache.hadoop.hive.ql.ddl.table.create;
import java.io.Serializable;
+import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import java.util.stream.Collectors;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.fs.Path;
@@ -34,6 +37,7 @@ import org.apache.hadoop.hive.metastore.TableType;
import org.apache.hadoop.hive.metastore.api.ColumnStatistics;
import org.apache.hadoop.hive.metastore.api.ColumnStatisticsDesc;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.metastore.api.ObjectDictionary;
import org.apache.hadoop.hive.metastore.api.Order;
import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
import org.apache.hadoop.hive.metastore.api.SQLCheckConstraint;
@@ -921,14 +925,23 @@ public class CreateTableDesc implements DDLDesc, Serializable {
// When replicating the statistics for a table will be obtained from the source. Do not
// reset it on replica.
if (replicationSpec == null || !replicationSpec.isInReplicationScope()) {
- if (!this.isCTAS && (tbl.getPath() == null || (!isExternal() && tbl.isEmpty()))) {
- if (!tbl.isPartitioned() && conf.getBoolVar(HiveConf.ConfVars.HIVESTATSAUTOGATHER)) {
- StatsSetupConst.setStatsStateForCreateTable(tbl.getTTable().getParameters(),
- MetaStoreUtils.getColumnNames(tbl.getCols()), StatsSetupConst.TRUE);
- }
- } else {
- StatsSetupConst.setStatsStateForCreateTable(tbl.getTTable().getParameters(), null,
- StatsSetupConst.FALSE);
+ // Remove COLUMN_STATS_ACCURATE=true from table's parameter, let the HMS determine if
+ // there is need to add column stats dependent on the table's location.
+ StatsSetupConst.setStatsStateForCreateTable(tbl.getTTable().getParameters(), null,
+ StatsSetupConst.FALSE);
+ if (!this.isCTAS && !tbl.isPartitioned() && !tbl.isTemporary() &&
+ conf.getBoolVar(HiveConf.ConfVars.HIVESTATSAUTOGATHER)) {
+ // Put the flag into the dictionary in order not to pollute the table,
+ // ObjectDictionary is meant to convey repeatitive messages.
+ ObjectDictionary dictionary = tbl.getTTable().isSetDictionary() ?
+ tbl.getTTable().getDictionary() : new ObjectDictionary();
+ List<ByteBuffer> buffers = new ArrayList<>();
+ String statsSetup = StatsSetupConst.ColumnStatsSetup.getStatsSetupAsString(true,
+ storageHandler != null && storageHandler.isMetadataTableSupported() ? "metadata" : null, // Skip metadata directory for Iceberg table
+ MetaStoreUtils.getColumnNames(tbl.getCols()));
+ buffers.add(ByteBuffer.wrap(statsSetup.getBytes(StandardCharsets.UTF_8)));
+ dictionary.putToValues(StatsSetupConst.STATS_FOR_CREATE_TABLE, buffers);
+ tbl.getTTable().setDictionary(dictionary);
}
}
diff --git a/ql/src/test/queries/clientpositive/stats_external_location.q b/ql/src/test/queries/clientpositive/stats_external_location.q
new file mode 100644
index 00000000000..87985a68d78
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/stats_external_location.q
@@ -0,0 +1,9 @@
+set hive.stats.column.autogather=true;
+set hive.stats.autogather=true;
+dfs ${system:test.dfs.mkdir} ${system:test.tmp.dir}/test1;
+
+create external table test_custom(age int, name string) stored as orc location '/tmp/test1';
+insert into test_custom select 1, 'test';
+desc formatted test_custom age;
+
+drop table test_custom;
diff --git a/ql/src/test/results/clientpositive/llap/default_file_format.q.out b/ql/src/test/results/clientpositive/llap/default_file_format.q.out
index 0adf5ae7415..df2621d7912 100644
--- a/ql/src/test/results/clientpositive/llap/default_file_format.q.out
+++ b/ql/src/test/results/clientpositive/llap/default_file_format.q.out
@@ -170,8 +170,13 @@ Retention: 0
#### A masked pattern was here ####
Table Type: EXTERNAL_TABLE
Table Parameters:
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"c\":\"true\"}}
EXTERNAL TRUE
bucketing_version 2
+ numFiles 0
+ numRows 0
+ rawDataSize 0
+ totalSize 0
#### A masked pattern was here ####
# Storage Information
@@ -234,9 +239,12 @@ Retention: 0
#### A masked pattern was here ####
Table Type: EXTERNAL_TABLE
Table Parameters:
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"c\":\"true\"}}
EXTERNAL TRUE
bucketing_version 2
numFiles 0
+ numRows 0
+ rawDataSize 0
totalSize 0
#### A masked pattern was here ####
@@ -470,9 +478,12 @@ Retention: 0
#### A masked pattern was here ####
Table Type: EXTERNAL_TABLE
Table Parameters:
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"c\":\"true\"}}
EXTERNAL TRUE
bucketing_version 2
numFiles 0
+ numRows 0
+ rawDataSize 0
totalSize 0
#### A masked pattern was here ####
@@ -536,9 +547,12 @@ Retention: 0
#### A masked pattern was here ####
Table Type: EXTERNAL_TABLE
Table Parameters:
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"c\":\"true\"}}
EXTERNAL TRUE
bucketing_version 2
numFiles 0
+ numRows 0
+ rawDataSize 0
totalSize 0
#### A masked pattern was here ####
diff --git a/ql/src/test/results/clientpositive/llap/mm_exim.q.out b/ql/src/test/results/clientpositive/llap/mm_exim.q.out
index c23d711534e..37ff35659c4 100644
--- a/ql/src/test/results/clientpositive/llap/mm_exim.q.out
+++ b/ql/src/test/results/clientpositive/llap/mm_exim.q.out
@@ -312,8 +312,8 @@ Table Type: MANAGED_TABLE
Table Parameters:
bucketing_version 2
numFiles 3
- numRows 0
- rawDataSize 0
+ numRows 6
+ rawDataSize 37
totalSize 43
transactional true
transactional_properties insert_only
diff --git a/ql/src/test/results/clientpositive/llap/stats_external_location.q.out b/ql/src/test/results/clientpositive/llap/stats_external_location.q.out
new file mode 100644
index 00000000000..f7fc782fbf3
--- /dev/null
+++ b/ql/src/test/results/clientpositive/llap/stats_external_location.q.out
@@ -0,0 +1,47 @@
+#### A masked pattern was here ####
+PREHOOK: type: CREATETABLE
+#### A masked pattern was here ####
+PREHOOK: Output: database:default
+PREHOOK: Output: default@test_custom
+#### A masked pattern was here ####
+POSTHOOK: type: CREATETABLE
+#### A masked pattern was here ####
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@test_custom
+PREHOOK: query: insert into test_custom select 1, 'test'
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@test_custom
+POSTHOOK: query: insert into test_custom select 1, 'test'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@test_custom
+POSTHOOK: Lineage: test_custom.age SIMPLE []
+POSTHOOK: Lineage: test_custom.name SIMPLE []
+PREHOOK: query: desc formatted test_custom age
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@test_custom
+POSTHOOK: query: desc formatted test_custom age
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@test_custom
+col_name age
+data_type int
+min 1
+max 1
+num_nulls 0
+distinct_count 1
+avg_col_len
+max_col_len
+num_trues
+num_falses
+bit_vector HL
+comment from deserializer
+COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"age\":\"true\",\"name\":\"true\"}}
+PREHOOK: query: drop table test_custom
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@test_custom
+PREHOOK: Output: default@test_custom
+POSTHOOK: query: drop table test_custom
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@test_custom
+POSTHOOK: Output: default@test_custom
diff --git a/ql/src/test/results/clientpositive/llap/translated_external_rename3.q.out b/ql/src/test/results/clientpositive/llap/translated_external_rename3.q.out
index c7c920d156b..ff321fa86f0 100644
--- a/ql/src/test/results/clientpositive/llap/translated_external_rename3.q.out
+++ b/ql/src/test/results/clientpositive/llap/translated_external_rename3.q.out
@@ -98,14 +98,11 @@ Retention: 0
#### A masked pattern was here ####
Table Type: EXTERNAL_TABLE
Table Parameters:
- COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"a\":\"true\"}}
EXTERNAL TRUE
TRANSLATED_TO_EXTERNAL TRUE
bucketing_version 2
external.table.purge TRUE
numFiles 2
- numRows 1
- rawDataSize 1
totalSize 4
#### A masked pattern was here ####
diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/common/StatsSetupConst.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/common/StatsSetupConst.java
index 0ee6bcfbfa2..7ca76bf3741 100644
--- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/common/StatsSetupConst.java
+++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/common/StatsSetupConst.java
@@ -167,6 +167,8 @@ public class StatsSetupConst {
public static final String CASCADE = "CASCADE";
+ public static final String STATS_FOR_CREATE_TABLE = "setStatsStateForCreateTable";
+
public static final String TRUE = "true";
public static final String FALSE = "false";
@@ -219,6 +221,55 @@ public class StatsSetupConst {
}
+ /**
+ * Class for marking the column statistics when creating tables.
+ */
+ public static class ColumnStatsSetup {
+ private static ObjectReader objectReader;
+ private static ObjectWriter objectWriter;
+ static {
+ ObjectMapper objectMapper = new ObjectMapper();
+ objectReader = objectMapper.readerFor(ColumnStatsSetup.class);
+ objectWriter = objectMapper.writerFor(ColumnStatsSetup.class);
+ }
+
+ @JsonInclude(JsonInclude.Include.NON_DEFAULT)
+ public boolean enabled;
+ @JsonInclude(JsonInclude.Include.NON_DEFAULT)
+ public String fileToEscape;
+ @JsonInclude(JsonInclude.Include.NON_EMPTY)
+ public List<String> columnNames = new ArrayList<>();
+
+ public static ColumnStatsSetup parseStatsSetup(String statsSetup) {
+ if (statsSetup == null) {
+ return new ColumnStatsSetup();
+ }
+ try {
+ return objectReader.readValue(statsSetup);
+ } catch (Exception e) {
+ return new ColumnStatsSetup();
+ }
+ }
+
+ /**
+ * Get json representation of the ColumnStatsSetup
+ */
+ public static String getStatsSetupAsString(boolean enabled,
+ String fileToEscape,
+ List<String> columns) {
+ try {
+ ColumnStatsSetup statsSetup = new ColumnStatsSetup();
+ statsSetup.enabled = enabled;
+ statsSetup.columnNames = new ArrayList<>(columns);
+ statsSetup.fileToEscape = fileToEscape;
+ return objectWriter.writeValueAsString(statsSetup);
+ } catch (Exception e) {
+ // this should not happen
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
public static boolean areBasicStatsUptoDate(Map<String, String> params) {
if (params == null) {
return false;
diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/Warehouse.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/Warehouse.java
index 10c9fb26d22..2952276020c 100755
--- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/Warehouse.java
+++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/Warehouse.java
@@ -32,6 +32,7 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.fs.PathFilter;
import org.apache.hadoop.hive.common.TableName;
import org.apache.hadoop.hive.metastore.api.Catalog;
import org.apache.hadoop.hive.metastore.api.DatabaseType;
@@ -500,16 +501,23 @@ public class Warehouse {
}
public boolean isEmptyDir(Path path) throws IOException, MetaException {
+ return isEmptyDir(path, null);
+ }
+
+ public boolean isEmptyDir(Path path, PathFilter pathFilter)
+ throws IOException, MetaException {
try {
- int listCount = getFs(path).listStatus(path).length;
- if (listCount == 0) {
- return true;
+ final int listCount;
+ if (pathFilter == null) {
+ listCount = getFs(path).listStatus(path).length;
+ } else {
+ listCount = getFs(path).listStatus(path, pathFilter).length;
}
+ return listCount == 0;
} catch (FileNotFoundException fnfe) {
// File named by path doesn't exist; nothing to validate.
return false;
}
- return false;
}
public boolean isWritable(Path path) throws IOException {
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HMSHandler.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HMSHandler.java
index d544bf30ed8..9033358a7cd 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HMSHandler.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HMSHandler.java
@@ -2361,10 +2361,8 @@ public class HMSHandler extends FacebookBase implements IHMSHandler {
madeDir = true;
}
}
- if (MetastoreConf.getBoolVar(conf, ConfVars.STATS_AUTO_GATHER) &&
- !MetaStoreUtils.isView(tbl)) {
- MetaStoreServerUtils.updateTableStatsSlow(db, tbl, wh, madeDir, false, envContext);
- }
+
+ MetaStoreServerUtils.updateTableStatsForCreateTable(wh, db, tbl, envContext, conf, tblPath, madeDir);
// set create time
long time = System.currentTimeMillis() / 1000;
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreServerUtils.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreServerUtils.java
index 6a15b089cd6..9ef97f0c578 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreServerUtils.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreServerUtils.java
@@ -27,6 +27,7 @@ import java.net.InetSocketAddress;
import java.net.ServerSocket;
import java.net.Socket;
import java.net.UnknownHostException;
+import java.nio.ByteBuffer;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.security.MessageDigest;
@@ -34,7 +35,7 @@ import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
-import java.util.Iterator;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
@@ -65,9 +66,11 @@ import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathFilter;
import org.apache.hadoop.hive.common.StatsSetupConst;
import org.apache.hadoop.hive.common.TableName;
import org.apache.hadoop.hive.metastore.ColumnType;
+import org.apache.hadoop.hive.metastore.ExceptionHandler;
import org.apache.hadoop.hive.metastore.HiveMetaStore;
import org.apache.hadoop.hive.metastore.IMetaStoreClient;
import org.apache.hadoop.hive.metastore.Warehouse;
@@ -510,6 +513,52 @@ public class MetaStoreServerUtils {
params.remove(StatsSetupConst.NUM_ERASURE_CODED_FILES);
}
+ public static void updateTableStatsForCreateTable(Warehouse wh, Database db, Table tbl,
+ EnvironmentContext envContext, Configuration conf, Path tblPath, boolean newDir)
+ throws MetaException {
+ // If the created table is a view, skip generating the stats
+ if (MetaStoreUtils.isView(tbl)) {
+ return;
+ }
+ assert tblPath != null;
+ if (tbl.isSetDictionary() && tbl.getDictionary().getValues() != null) {
+ List<ByteBuffer> values = tbl.getDictionary().getValues().
+ remove(StatsSetupConst.STATS_FOR_CREATE_TABLE);
+ ByteBuffer buffer;
+ if (values != null && values.size() > 0 && (buffer = values.get(0)).hasArray()) {
+ String val = new String(buffer.array(), StandardCharsets.UTF_8);
+ StatsSetupConst.ColumnStatsSetup statsSetup = StatsSetupConst.ColumnStatsSetup.parseStatsSetup(val);
+ if (statsSetup.enabled) {
+ try {
+ PathFilter pathFilter = FileUtils.HIDDEN_FILES_PATH_FILTER;
+ if (StringUtils.isNotEmpty(statsSetup.fileToEscape)) {
+ final Set<String> filesToEscape = new HashSet<>();
+ for (String fileName : statsSetup.fileToEscape.split(",")) {
+ filesToEscape.add(fileName.trim());
+ }
+ pathFilter = p -> !filesToEscape.contains(p.getName());
+ }
+ // Set the column stats true in order to make it merge-able
+ if (newDir || wh.isEmptyDir(tblPath, pathFilter)) {
+ List<String> columns = statsSetup.columnNames;
+ if (columns == null || columns.isEmpty()) {
+ columns = getColumnNames(tbl.getSd().getCols());
+ }
+ StatsSetupConst.setStatsStateForCreateTable(tbl.getParameters(), columns, StatsSetupConst.TRUE);
+ }
+ } catch (IOException e) {
+ LOG.error("Error while checking the table directory: " + tblPath + " is empty or not", e);
+ throw ExceptionHandler.newMetaException(e);
+ }
+ }
+ }
+ }
+
+ if (MetastoreConf.getBoolVar(conf, MetastoreConf.ConfVars.STATS_AUTO_GATHER)) {
+ updateTableStatsSlow(db, tbl, wh, newDir, false, envContext);
+ }
+ }
+
/**
* Compare the names, types and comments of two lists of {@link FieldSchema}.
* <p>