You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by jo...@apache.org on 2022/01/06 23:19:41 UTC
[impala] branch master updated (1ed48a5 -> 351e037)
This is an automated email from the ASF dual-hosted git repository.
joemcdonnell pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git.
from 1ed48a5 IMPALA-11005 (part 3): Repalce random number generator with mt19937_64
new 7131ee3 IMPALA-10934 (Part 1): Enable table definition over a single file
new 351e037 IMPALA-10934 (Part 2): Enable table definition over a single file
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
be/src/exec/hdfs-table-sink.cc | 1 +
be/src/runtime/io/disk-io-mgr-test.cc | 4 +-
be/src/runtime/io/disk-io-mgr.cc | 8 +
be/src/runtime/io/disk-io-mgr.h | 4 +
be/src/util/hdfs-util.cc | 5 +
be/src/util/hdfs-util.h | 3 +
bin/rat_exclude_files.txt | 2 +
.../org/apache/impala/common/FileSystemUtil.java | 7 +-
java/shaded-deps/hive-exec/pom.xml | 2 +
testdata/data/sfs_d1.parq | Bin 0 -> 544 bytes
testdata/data/sfs_d2.txt | 5 +
testdata/data/sfs_d3.parq | Bin 0 -> 544 bytes
testdata/data/sfs_d4.txt | 5 +
.../functional-query/queries/QueryTest/sfs.test | 286 +++++++++++++++++++++
tests/common/file_utils.py | 16 +-
tests/query_test/test_sfs.py | 52 ++++
16 files changed, 392 insertions(+), 8 deletions(-)
create mode 100644 testdata/data/sfs_d1.parq
create mode 100644 testdata/data/sfs_d2.txt
create mode 100644 testdata/data/sfs_d3.parq
create mode 100644 testdata/data/sfs_d4.txt
create mode 100644 testdata/workloads/functional-query/queries/QueryTest/sfs.test
create mode 100644 tests/query_test/test_sfs.py
[impala] 02/02: IMPALA-10934 (Part 2): Enable table definition over a single file
Posted by jo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
joemcdonnell pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 351e0374727be7ccef6843cf1036fd6168220d3e
Author: Fang-Yu Rao <fa...@cloudera.com>
AuthorDate: Wed Nov 17 12:48:40 2021 -0800
IMPALA-10934 (Part 2): Enable table definition over a single file
This patch adds an end-to-end test to validate and characterize HMS'
behavior with respect to external table creation after HIVE-25569 via
which a user is allowed to create an external table associated with a
single file.
Change-Id: Ia4f57f07a9f543c660b102ebf307a6cf590a6784
Reviewed-on: http://gerrit.cloudera.org:8080/18033
Tested-by: Impala Public Jenkins <im...@cloudera.com>
Reviewed-by: Aman Sinha <am...@cloudera.com>
---
bin/rat_exclude_files.txt | 2 +
testdata/data/sfs_d1.parq | Bin 0 -> 544 bytes
testdata/data/sfs_d2.txt | 5 +
testdata/data/sfs_d3.parq | Bin 0 -> 544 bytes
testdata/data/sfs_d4.txt | 5 +
.../functional-query/queries/QueryTest/sfs.test | 286 +++++++++++++++++++++
tests/common/file_utils.py | 16 +-
tests/query_test/test_sfs.py | 52 ++++
8 files changed, 360 insertions(+), 6 deletions(-)
diff --git a/bin/rat_exclude_files.txt b/bin/rat_exclude_files.txt
index aef14f3..32c12fd 100644
--- a/bin/rat_exclude_files.txt
+++ b/bin/rat_exclude_files.txt
@@ -154,6 +154,8 @@ testdata/data/widerow.txt
testdata/data/local_tbl/00000.txt
testdata/data/hudi_parquet/*
testdata/data/iceberg_test/*
+testdata/data/sfs_d2.txt
+testdata/data/sfs_d4.txt
testdata/datasets/functional/functional_schema_template.sql
testdata/impala-profiles/README
testdata/impala-profiles/impala_profile_log_tpcds_compute_stats
diff --git a/testdata/data/sfs_d1.parq b/testdata/data/sfs_d1.parq
new file mode 100644
index 0000000..5ff07b6
Binary files /dev/null and b/testdata/data/sfs_d1.parq differ
diff --git a/testdata/data/sfs_d2.txt b/testdata/data/sfs_d2.txt
new file mode 100644
index 0000000..0eb236c
--- /dev/null
+++ b/testdata/data/sfs_d2.txt
@@ -0,0 +1,5 @@
+u,1
+v,2
+w,3
+x,4
+y,5
diff --git a/testdata/data/sfs_d3.parq b/testdata/data/sfs_d3.parq
new file mode 100644
index 0000000..5ff07b6
Binary files /dev/null and b/testdata/data/sfs_d3.parq differ
diff --git a/testdata/data/sfs_d4.txt b/testdata/data/sfs_d4.txt
new file mode 100644
index 0000000..0eb236c
--- /dev/null
+++ b/testdata/data/sfs_d4.txt
@@ -0,0 +1,5 @@
+u,1
+v,2
+w,3
+x,4
+y,5
diff --git a/testdata/workloads/functional-query/queries/QueryTest/sfs.test b/testdata/workloads/functional-query/queries/QueryTest/sfs.test
new file mode 100644
index 0000000..20ba813
--- /dev/null
+++ b/testdata/workloads/functional-query/queries/QueryTest/sfs.test
@@ -0,0 +1,286 @@
+====
+---- QUERY
+# We do not hardcode the host name to something like "localhost" since the host name may
+# be an IP address in a test environment.
+CREATE EXTERNAL TABLE test_tbl_01 (s STRING, i INT) STORED AS PARQUET
+LOCATION 'sfs+hdfs://$INTERNAL_LISTEN_HOST:20500/test-warehouse/$DATABASE.db/sfs_d1.parq/#SINGLEFILE#'
+---- RESULTS
+'Table has been created.'
+====
+---- QUERY
+CREATE EXTERNAL TABLE test_tbl_02 (s STRING, i INT)
+ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
+STORED AS TEXTFILE
+LOCATION 'sfs+hdfs://$INTERNAL_LISTEN_HOST:20500/test-warehouse/$DATABASE.db/sfs_d2.txt/#SINGLEFILE#'
+---- RESULTS
+'Table has been created.'
+====
+---- QUERY
+SELECT * FROM test_tbl_01;
+---- RESULTS
+'a',1
+'b',2
+'c',3
+'d',4
+'e',5
+---- TYPES
+STRING, INT
+====
+---- QUERY
+SELECT * FROM test_tbl_02;
+---- RESULTS
+'u',1
+'v',2
+'w',3
+'x',4
+'y',5
+---- TYPES
+STRING, INT
+====
+---- QUERY
+INSERT INTO TABLE test_tbl_01 VALUES ('x', 100);
+---- CATCH
+row_regex: .*Unable to INSERT into target table .+ because .+ is not a supported filesystem.*
+====
+---- QUERY
+INSERT INTO TABLE test_tbl_02 VALUES ('x', 100);
+---- CATCH
+row_regex: .*Unable to INSERT into target table .+ because .+ is not a supported filesystem.*
+====
+---- QUERY
+LOAD DATA INPATH 'hdfs://$INTERNAL_LISTEN_HOST:20500/test-warehouse/$DATABASE.db/sfs_d3.parq' INTO TABLE test_tbl_01
+---- CATCH
+Unsupported SFS filesystem operation!
+====
+---- QUERY
+LOAD DATA INPATH 'hdfs://$INTERNAL_LISTEN_HOST:20500/test-warehouse/$DATABASE.db/sfs_d4.txt' INTO TABLE test_tbl_02
+---- CATCH
+Unsupported SFS filesystem operation!
+====
+---- QUERY
+LOAD DATA INPATH 'sfs+hdfs://$INTERNAL_LISTEN_HOST:20500/test-warehouse/$DATABASE.db/sfs_d3.parq/#SINGLEFILE#' INTO TABLE test_tbl_01
+---- CATCH
+row_regex: .*INPATH location .+ must point to one of the supported filesystem URI scheme.*
+====
+---- QUERY
+LOAD DATA INPATH 'sfs+hdfs://$INTERNAL_LISTEN_HOST:20500/test-warehouse/$DATABASE.db/sfs_d4.txt/#SINGLEFILE#' INTO TABLE test_tbl_02
+---- CATCH
+row_regex: .*INPATH location .+ must point to one of the supported filesystem URI scheme.*
+====
+---- QUERY
+TRUNCATE TABLE $DATABASE.test_tbl_01
+---- CATCH
+Unsupported SFS filesystem operation!
+====
+---- QUERY
+TRUNCATE TABLE $DATABASE.test_tbl_02
+---- CATCH
+Unsupported SFS filesystem operation!
+====
+---- QUERY
+COMPUTE STATS $DATABASE.test_tbl_01
+---- RESULTS
+'Updated 1 partition(s) and 2 column(s).'
+====
+---- QUERY
+COMPUTE STATS $DATABASE.test_tbl_02
+---- RESULTS
+'Updated 1 partition(s) and 2 column(s).'
+====
+---- QUERY
+CREATE EXTERNAL TABLE test_tbl_03 (s STRING, i INT) STORED AS PARQUET
+LOCATION 'sfs+hdfs://$INTERNAL_LISTEN_HOST:20500/test-warehouse/managed/$DATABASE.db/sfs_d3.parq/#SINGLEFILE#'
+---- CATCH
+An external table's location should not be located within managed warehouse root directory
+====
+---- QUERY
+CREATE EXTERNAL TABLE test_tbl_04 (s STRING, i INT)
+ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
+STORED AS TEXTFILE
+LOCATION 'sfs+hdfs://$INTERNAL_LISTEN_HOST:20500/test-warehouse/managed/$DATABASE.db/sfs_d4.txt/#SINGLEFILE#'
+---- CATCH
+An external table's location should not be located within managed warehouse root directory
+====
+---- QUERY
+# The table can actually be created.
+CREATE TABLE test_tbl_03 (s STRING, i INT)
+STORED AS PARQUET
+LOCATION 'sfs+hdfs://$INTERNAL_LISTEN_HOST:20500/test-warehouse/managed/$DATABASE.db/sfs_d3.parq/#SINGLEFILE#'
+---- RESULTS
+'Table has been created.'
+====
+---- QUERY
+# The table can actually be created.
+CREATE TABLE test_tbl_04 (s STRING, i INT)
+ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
+STORED AS TEXTFILE
+LOCATION 'sfs+hdfs://$INTERNAL_LISTEN_HOST:20500/test-warehouse/managed/$DATABASE.db/sfs_d4.txt/#SINGLEFILE#'
+---- RESULTS
+'Table has been created.'
+====
+---- QUERY
+# test_tbl_03 is considered as an external table with the parameter
+# 'external.table.purge' set to true. But the associated single file will not be deleted
+# after the table is dropped.
+DESCRIBE FORMATTED test_tbl_03
+---- RESULTS: VERIFY_IS_SUBSET
+'Table Type: ','EXTERNAL_TABLE ','NULL'
+'','external.table.purge','TRUE '
+'','EXTERNAL ','TRUE '
+---- TYPES
+STRING,STRING,STRING
+====
+---- QUERY
+# test_tbl_04 is considered as an external table with the parameter
+# 'external.table.purge' set to true. But the associated single file will not be deleted
+# after the table is dropped.
+DESCRIBE FORMATTED test_tbl_04
+---- RESULTS: VERIFY_IS_SUBSET
+'Table Type: ','EXTERNAL_TABLE ','NULL'
+'','external.table.purge','TRUE '
+'','EXTERNAL ','TRUE '
+---- TYPES
+STRING,STRING,STRING
+====
+---- QUERY
+# The contents of the table can be correctly retrieved.
+SELECT * FROM test_tbl_03;
+---- RESULTS
+'a',1
+'b',2
+'c',3
+'d',4
+'e',5
+---- TYPES
+STRING, INT
+====
+---- QUERY
+# The contents of the table can be correctly retrieved.
+SELECT * FROM test_tbl_04;
+---- RESULTS
+'u',1
+'v',2
+'w',3
+'x',4
+'y',5
+---- TYPES
+STRING, INT
+====
+---- QUERY
+INSERT INTO TABLE test_tbl_03 VALUES ('x', 100);
+---- CATCH
+row_regex: .*Unable to INSERT into target table .+ because .+ is not a supported filesystem.*
+====
+---- QUERY
+INSERT INTO TABLE test_tbl_04 VALUES ('x', 100);
+---- CATCH
+row_regex: .*Unable to INSERT into target table .+ because .+ is not a supported filesystem.*
+====
+---- QUERY
+DROP TABLE test_tbl_03;
+DROP TABLE test_tbl_04
+====
+---- QUERY
+# test_tbl_05 can be created, which shows that sfs_d3.parq has not been deleted after
+# test_tbl_03 was dropped.
+CREATE TABLE test_tbl_05 (s STRING, i INT) STORED AS PARQUET
+LOCATION 'sfs+hdfs://$INTERNAL_LISTEN_HOST:20500/test-warehouse/managed/$DATABASE.db/sfs_d3.parq/#SINGLEFILE#'
+---- RESULTS
+'Table has been created.'
+====
+---- QUERY
+# test_tbl_06 can be created, which shows that sfs_d4.txt has not been deleted after
+# test_tbl_04 was dropped.
+CREATE TABLE test_tbl_06 (s STRING, i INT)
+ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
+STORED AS TEXTFILE
+LOCATION 'sfs+hdfs://$INTERNAL_LISTEN_HOST:20500/test-warehouse/managed/$DATABASE.db/sfs_d4.txt/#SINGLEFILE#'
+---- RESULTS
+'Table has been created.'
+====
+---- QUERY
+# The contents of the table can still be correctly retrieved.
+SELECT * FROM test_tbl_05;
+---- RESULTS
+'a',1
+'b',2
+'c',3
+'d',4
+'e',5
+---- TYPES
+STRING, INT
+====
+---- QUERY
+# The contents of the table can be still correctly retrieved.
+SELECT * FROM test_tbl_06;
+---- RESULTS
+'u',1
+'v',2
+'w',3
+'x',4
+'y',5
+---- TYPES
+STRING, INT
+====
+---- QUERY
+# We need to set the following query option to submit to HMS a request to create a
+# managed table.
+SET DEFAULT_TRANSACTIONAL_TYPE=INSERT_ONLY;
+CREATE TABLE test_tbl_03 (s STRING, i INT)
+STORED AS PARQUET
+LOCATION 'sfs+hdfs://$INTERNAL_LISTEN_HOST:20500/test-warehouse/managed/$DATABASE.db/sfs_d3.parq/#SINGLEFILE#'
+---- CATCH
+A managed table's location should be located within managed warehouse root directory or within its database's managedLocationUri.
+====
+---- QUERY
+# We need to set the following query option to submit to HMS a request to create a
+# managed table.
+SET DEFAULT_TRANSACTIONAL_TYPE=INSERT_ONLY;
+CREATE TABLE test_tbl_04 (s STRING, i INT)
+ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
+STORED AS TEXTFILE
+LOCATION 'sfs+hdfs://$INTERNAL_LISTEN_HOST:20500/test-warehouse/managed/$DATABASE.db/sfs_d4.txt/#SINGLEFILE#'
+---- CATCH
+A managed table's location should be located within managed warehouse root directory or within its database's managedLocationUri.
+====
+---- QUERY
+SET DEFAULT_TRANSACTIONAL_TYPE=INSERT_ONLY;
+CREATE TABLE test_tbl_03 (s STRING, i INT)
+STORED AS PARQUET
+LOCATION 'hdfs://$INTERNAL_LISTEN_HOST:20500/test-warehouse/managed/$DATABASE.db/sfs_d3.parq'
+---- CATCH
+Path is not a directory
+====
+---- QUERY
+SET DEFAULT_TRANSACTIONAL_TYPE=INSERT_ONLY;
+CREATE TABLE test_tbl_04 (s STRING, i INT)
+ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
+STORED AS TEXTFILE
+LOCATION 'hdfs://$INTERNAL_LISTEN_HOST:20500/test-warehouse/managed/$DATABASE.db/sfs_d4.txt'
+---- CATCH
+Path is not a directory
+====
+---- QUERY
+# The table can actually be created but the contents of the table cannot be retrieved.
+CREATE EXTERNAL TABLE test_tbl_03 (s STRING) PARTITIONED BY (i INT) STORED AS PARQUET
+LOCATION 'sfs+hdfs://$INTERNAL_LISTEN_HOST:20500/test-warehouse/$DATABASE.db/sfs_d3.parq/#SINGLEFILE#'
+---- RESULTS
+'Table has been created.'
+====
+---- QUERY
+# The table can actually be created but the contents of the table cannot be retrieved.
+CREATE EXTERNAL TABLE test_tbl_04 (s STRING) PARTITIONED BY (i INT)
+ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
+STORED AS TEXTFILE
+LOCATION 'sfs+hdfs://$INTERNAL_LISTEN_HOST:20500/test-warehouse/$DATABASE.db/sfs_d4.txt/#SINGLEFILE#'
+---- RESULTS
+'Table has been created.'
+====
+---- QUERY
+SELECT * FROM test_tbl_03
+---- RESULTS
+====
+---- QUERY
+SELECT * FROM test_tbl_04
+---- RESULTS
+====
diff --git a/tests/common/file_utils.py b/tests/common/file_utils.py
index 529f72d..2de5549 100644
--- a/tests/common/file_utils.py
+++ b/tests/common/file_utils.py
@@ -65,12 +65,20 @@ def create_table_and_copy_files(impala_client, create_stmt, unique_database, tab
create_stmt = create_stmt.format(db=unique_database, tbl=table_name)
impala_client.execute(create_stmt)
+ hdfs_dir = get_fs_path(
+ os.path.join("/test-warehouse", unique_database + ".db", table_name))
+ copy_files_to_hdfs_dir(files, hdfs_dir)
+
+ # Refresh the table metadata to see the new files
+ refresh_stmt = "refresh {0}.{1}".format(unique_database, table_name)
+ impala_client.execute(refresh_stmt)
+
+
+def copy_files_to_hdfs_dir(files, hdfs_dir):
# Copy the files
# - build a list of source files
# - issue a single put to the hdfs_dir ( -d skips a staging copy)
source_files = []
- hdfs_dir = get_fs_path(
- os.path.join("/test-warehouse", unique_database + ".db", table_name))
for local_file in files:
# Cut off leading '/' to make os.path.join() happy
local_file = local_file if local_file[0] != '/' else local_file[1:]
@@ -79,10 +87,6 @@ def create_table_and_copy_files(impala_client, create_stmt, unique_database, tab
source_files.append(local_file)
check_call(['hdfs', 'dfs', '-put', '-f', '-d'] + source_files + [hdfs_dir])
- # Refresh the table metadata to see the new files
- refresh_stmt = "refresh {0}.{1}".format(unique_database, table_name)
- impala_client.execute(refresh_stmt)
-
def grep_dir(dir, search, filename_search=""):
'''Recursively search for files that contain 'search' and have a filename that matches
diff --git a/tests/query_test/test_sfs.py b/tests/query_test/test_sfs.py
new file mode 100644
index 0000000..3973a1c
--- /dev/null
+++ b/tests/query_test/test_sfs.py
@@ -0,0 +1,52 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# This test suite validates the functionality that allows users to create an external
+# table associated with a single file.
+
+from tests.common.file_utils import copy_files_to_hdfs_dir
+from tests.common.impala_test_suite import ImpalaTestSuite
+from tests.common.skip import SkipIf
+
+
+class TestSFS(ImpalaTestSuite):
+ @classmethod
+ def get_workload(cls):
+ return 'functional-query'
+
+ @classmethod
+ def add_test_dimensions(cls):
+ super(TestSFS, cls).add_test_dimensions()
+ cls.ImpalaTestMatrix.add_constraint(lambda v:
+ v.get_value('table_format').file_format == 'text' and
+ v.get_value('table_format').compression_codec == 'none')
+ cls.ImpalaTestMatrix.add_constraint(lambda v:
+ v.get_value('exec_option')['disable_codegen'] is False)
+
+ @SkipIf.not_hdfs
+ def test_sfs(self, vector, unique_database):
+ files_for_external_tables = ["testdata/data/sfs_d1.parq", "testdata/data/sfs_d2.txt",
+ "testdata/data/sfs_d3.parq", "testdata/data/sfs_d4.txt"]
+ files_for_managed_tables = ["testdata/data/sfs_d3.parq", "testdata/data/sfs_d4.txt"]
+ hdfs_dir_for_external_tables = "/test-warehouse/{0}.db/".format(unique_database)
+ hdfs_dir_for_managed_tables =\
+ "/test-warehouse/managed/{0}.db/".format(unique_database)
+
+ copy_files_to_hdfs_dir(files_for_external_tables, hdfs_dir_for_external_tables)
+ copy_files_to_hdfs_dir(files_for_managed_tables, hdfs_dir_for_managed_tables)
+
+ self.run_test_case('QueryTest/sfs', vector, unique_database)
[impala] 01/02: IMPALA-10934 (Part 1): Enable table definition over a single file
Posted by jo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
joemcdonnell pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 7131ee34d25b2609c4eb0cc596bc1d99d6232acd
Author: Zoltan Haindrich <ki...@rxd.hu>
AuthorDate: Wed Nov 3 19:07:58 2021 +0100
IMPALA-10934 (Part 1): Enable table definition over a single file
Implements an abstraction layer to show files in a single directory.
Impala side part - filesystem drivers are in HIVE-25569.
Suppose that the filesystem has a directory in which there are multiple
files:
hdfs://somedir/f1.txt
hdfs://somedir/f2.txt
In case of a HMS backed table(s) - the contents of a directory could be
considered as table.
This patch enables a new file system wrapper 'sfs+' (sfs = single file
system) which provides a view of a single file in a directory.' The '+'
indicates that this wrapper can be added on top of multiple underlying
file systems/object storage such as HDFS, S3 etc. The directory which
contains the file could be specified:
sfs+hdfs://somedir/f1.txt/#SINGLEFILE#
This will be a directory containing only the f1.txt and nothing else.
This patch was tested locally - with a custom build of Hive version
which also had HIVE-25569.
Change-Id: I32be936243aa4c8320f5d06d2b7fbf98822f82e7
Reviewed-on: http://gerrit.cloudera.org:8080/17878
Reviewed-by: Aman Sinha <am...@cloudera.com>
Tested-by: Aman Sinha <am...@cloudera.com>
---
be/src/exec/hdfs-table-sink.cc | 1 +
be/src/runtime/io/disk-io-mgr-test.cc | 4 +++-
be/src/runtime/io/disk-io-mgr.cc | 8 ++++++++
be/src/runtime/io/disk-io-mgr.h | 4 ++++
be/src/util/hdfs-util.cc | 5 +++++
be/src/util/hdfs-util.h | 3 +++
fe/src/main/java/org/apache/impala/common/FileSystemUtil.java | 7 ++++++-
java/shaded-deps/hive-exec/pom.xml | 2 ++
8 files changed, 32 insertions(+), 2 deletions(-)
diff --git a/be/src/exec/hdfs-table-sink.cc b/be/src/exec/hdfs-table-sink.cc
index b65e338..8a72d6c 100644
--- a/be/src/exec/hdfs-table-sink.cc
+++ b/be/src/exec/hdfs-table-sink.cc
@@ -461,6 +461,7 @@ Status HdfsTableSink::CreateNewTmpFile(RuntimeState* state,
IsADLSPath(tmp_hdfs_file_name_cstr) ||
IsGcsPath(tmp_hdfs_file_name_cstr) ||
IsCosPath(tmp_hdfs_file_name_cstr) ||
+ IsSFSPath(tmp_hdfs_file_name_cstr) ||
IsOzonePath(tmp_hdfs_file_name_cstr)) {
// On S3A, the file cannot be stat'ed until after it's closed, and even so, the block
// size reported will be just the filesystem default. Similarly, the block size
diff --git a/be/src/runtime/io/disk-io-mgr-test.cc b/be/src/runtime/io/disk-io-mgr-test.cc
index f491f59..4dab7af 100644
--- a/be/src/runtime/io/disk-io-mgr-test.cc
+++ b/be/src/runtime/io/disk-io-mgr-test.cc
@@ -58,6 +58,7 @@ DECLARE_int32(num_ozone_io_threads);
DECLARE_int32(num_oss_io_threads);
DECLARE_int32(num_remote_hdfs_file_oper_io_threads);
DECLARE_int32(num_s3_file_oper_io_threads);
+DECLARE_int32(num_sfs_io_threads);
#ifndef NDEBUG
DECLARE_int32(stress_disk_read_delay_ms);
@@ -1719,7 +1720,8 @@ TEST_F(DiskIoMgrTest, VerifyNumThreadsParameter) {
+ FLAGS_num_ozone_io_threads + FLAGS_num_oss_io_threads
+ FLAGS_num_remote_hdfs_file_oper_io_threads
+ FLAGS_num_s3_file_oper_io_threads + FLAGS_num_gcs_io_threads
- + FLAGS_num_cos_io_threads;
+ + FLAGS_num_cos_io_threads
+ + FLAGS_num_sfs_io_threads;
// Verify num_io_threads_per_rotational_disk and num_io_threads_per_solid_state_disk.
// Since we do not have control over which disk is used, we check for either type
diff --git a/be/src/runtime/io/disk-io-mgr.cc b/be/src/runtime/io/disk-io-mgr.cc
index aef83f9..22bead3 100644
--- a/be/src/runtime/io/disk-io-mgr.cc
+++ b/be/src/runtime/io/disk-io-mgr.cc
@@ -142,6 +142,9 @@ DEFINE_int32(num_cos_io_threads, 16, "Number of COS I/O threads");
// The maximum number of Ozone I/O threads. TODO: choose the default empirically.
DEFINE_int32(num_ozone_io_threads, 16, "Number of Ozone I/O threads");
+// The maximum number of SFS I/O threads.
+DEFINE_int32(num_sfs_io_threads, 16, "Number of SFS I/O threads");
+
// The number of cached file handles defines how much memory can be used per backend for
// caching frequently used file handles. Measurements indicate that a single file handle
// uses about 6kB of memory. 20k file handles will thus reserve ~120MB of memory.
@@ -492,6 +495,9 @@ Status DiskIoMgr::Init() {
} else if (i == RemoteS3DiskFileOperId()) {
num_threads_per_disk = FLAGS_num_s3_file_oper_io_threads;
device_name = "S3 remote file operations";
+ } else if (i == RemoteSFSDiskId()) {
+ num_threads_per_disk = FLAGS_num_sfs_io_threads;
+ device_name = "SFS remote";
} else if (DiskInfo::is_rotational(i)) {
num_threads_per_disk = num_io_threads_per_rotational_disk_;
// During tests, i may not point to an existing disk.
@@ -825,6 +831,7 @@ int DiskIoMgr::AssignQueue(
if (IsGcsPath(file, check_default_fs)) return RemoteGcsDiskId();
if (IsCosPath(file, check_default_fs)) return RemoteCosDiskId();
if (IsOzonePath(file, check_default_fs)) return RemoteOzoneDiskId();
+ if (IsSFSPath(file, check_default_fs)) return RemoteSFSDiskId();
}
// Assign to a local disk queue.
DCHECK(!IsS3APath(file, check_default_fs)); // S3 is always remote.
@@ -834,6 +841,7 @@ int DiskIoMgr::AssignQueue(
DCHECK(!IsGcsPath(file, check_default_fs)); // GCS is always remote.
DCHECK(!IsCosPath(file, check_default_fs)); // COS is always remote.
DCHECK(!IsOzonePath(file, check_default_fs)); // Ozone is always remote.
+ DCHECK(!IsSFSPath(file, check_default_fs)); // SFS is always remote.
if (disk_id == -1) {
// disk id is unknown, assign it an arbitrary one.
disk_id = next_disk_id_.Add(1);
diff --git a/be/src/runtime/io/disk-io-mgr.h b/be/src/runtime/io/disk-io-mgr.h
index 9495673..95bf0e8 100644
--- a/be/src/runtime/io/disk-io-mgr.h
+++ b/be/src/runtime/io/disk-io-mgr.h
@@ -339,6 +339,9 @@ class DiskIoMgr : public CacheLineAligned {
/// The disk ID (and therefore disk_queues_ index) used for Ozone accesses.
int RemoteOzoneDiskId() const { return num_local_disks() + REMOTE_OZONE_DISK_OFFSET; }
+ /// The disk ID (and therefore disk_queues_ index) used for SFS accesses.
+ int RemoteSFSDiskId() const { return num_local_disks() + REMOTE_SFS_DISK_OFFSET; }
+
/// Dumps the disk IoMgr queues (for readers and disks)
std::string DebugString();
@@ -397,6 +400,7 @@ class DiskIoMgr : public CacheLineAligned {
REMOTE_OZONE_DISK_OFFSET,
REMOTE_DFS_DISK_FILE_OPER_OFFSET,
REMOTE_S3_DISK_FILE_OPER_OFFSET,
+ REMOTE_SFS_DISK_OFFSET,
REMOTE_OSS_DISK_OFFSET,
REMOTE_NUM_DISKS
};
diff --git a/be/src/util/hdfs-util.cc b/be/src/util/hdfs-util.cc
index 456dd8a..1d6db5f 100644
--- a/be/src/util/hdfs-util.cc
+++ b/be/src/util/hdfs-util.cc
@@ -38,6 +38,7 @@ const char* FILESYS_PREFIX_GCS = "gs://";
const char* FILESYS_PREFIX_COS = "cosn://";
const char* FILESYS_PREFIX_OZONE = "o3fs://";
const char* FILESYS_PREFIX_OFS = "ofs://";
+const char* FILESYS_PREFIX_SFS = "sfs+";
const char* FILESYS_PREFIX_OSS = "oss://";
const char* FILESYS_PREFIX_JINDOFS = "jfs://";
@@ -133,6 +134,10 @@ bool IsOzonePath(const char* path, bool check_default_fs) {
|| IsSpecificPath(path, FILESYS_PREFIX_OFS, check_default_fs);
}
+bool IsSFSPath(const char* path, bool check_default_fs) {
+ return IsSpecificPath(path, FILESYS_PREFIX_SFS, check_default_fs);
+}
+
// Returns the length of the filesystem name in 'path' which is the length of the
// 'scheme://authority'. Returns 0 if the path is unqualified.
static int GetFilesystemNameLength(const char* path) {
diff --git a/be/src/util/hdfs-util.h b/be/src/util/hdfs-util.h
index 8de3fcc..a31a76d 100644
--- a/be/src/util/hdfs-util.h
+++ b/be/src/util/hdfs-util.h
@@ -80,6 +80,9 @@ bool IsCosPath(const char* path, bool check_default_fs = true);
/// Returns true iff the path refers to a location on an Ozone filesystem.
bool IsOzonePath(const char* path, bool check_default_fs = true);
+/// Returns true iff the path refers to a location on an SFS filesystem.
+bool IsSFSPath(const char* path, bool check_default_fs = true);
+
/// Returns true iff 'pathA' and 'pathB' are on the same filesystem.
bool FilesystemsMatch(const char* pathA, const char* pathB);
diff --git a/fe/src/main/java/org/apache/impala/common/FileSystemUtil.java b/fe/src/main/java/org/apache/impala/common/FileSystemUtil.java
index 09a353a..e5439d6 100644
--- a/fe/src/main/java/org/apache/impala/common/FileSystemUtil.java
+++ b/fe/src/main/java/org/apache/impala/common/FileSystemUtil.java
@@ -69,6 +69,7 @@ public class FileSystemUtil {
public static final String SCHEME_ALLUXIO = "alluxio";
public static final String SCHEME_GCS = "gs";
public static final String SCHEME_COS = "cosn";
+ public static final String SCHEME_SFS = "sfs";
/**
* Set containing all FileSystem scheme that known to supports storage UUIDs in
@@ -531,7 +532,8 @@ public class FileSystemUtil {
OZONE,
ALLUXIO,
GCS,
- COS;
+ COS,
+ SFS;
private static final Map<String, FsType> SCHEME_TO_FS_MAPPING =
ImmutableMap.<String, FsType>builder()
@@ -556,6 +558,9 @@ public class FileSystemUtil {
* hdfs, s3a, etc.)
*/
public static FsType getFsType(String scheme) {
+ if(scheme.startsWith("sfs+")) {
+ return SFS;
+ }
return SCHEME_TO_FS_MAPPING.get(scheme);
}
}
diff --git a/java/shaded-deps/hive-exec/pom.xml b/java/shaded-deps/hive-exec/pom.xml
index 7fd6a72..58956c9 100644
--- a/java/shaded-deps/hive-exec/pom.xml
+++ b/java/shaded-deps/hive-exec/pom.xml
@@ -89,6 +89,8 @@ the same dependencies
<filter>
<artifact>org.apache.hive:hive-exec</artifact>
<includes>
+ <include>META-INF/services/org.apache.hadoop.fs.FileSystem</include>
+ <include>org/apache/hadoop/hive/ql/io/SingleFileSystem*</include>
<include>org/apache/hadoop/hive/conf/**/*</include>
<include>org/apache/hadoop/hive/common/type/*</include>
<include>org/apache/hadoop/hive/common/FileUtils*</include>