You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mo...@apache.org on 2023/11/03 05:42:37 UTC
(doris) branch master updated: [Fix](multi-catalog) Filter _temporary dir which is generated by spark. (#26194)
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 9019a71770e [Fix](multi-catalog) Filter _temporary dir which is generated by spark. (#26194)
9019a71770e is described below
commit 9019a71770ea158b035159bb523a4501aee87b55
Author: Xiangyu Wang <du...@gmail.com>
AuthorDate: Fri Nov 3 13:42:29 2023 +0800
[Fix](multi-catalog) Filter _temporary dir which is generated by spark. (#26194)
Sometimes there are some temporary files which are generated by some spark jobs, we need filter these files.
```
errCode = 2, detailMessage = (xxx.xxx.xxx.xxx.xxx)
[CANCELLED][INTERNAL_ERROR]failed to init reader for file hdfs://xxx/user/hive/warehouse/xxx.db/xxx/_temporary/0/_temporary/attempt_202311010952379045223848154629697_0013_m_000442_3355/part-00442-3f97ac18-0583-4d2c-8995-8d21ceda56ec-c000.snappy.orc,
err: [INTERNAL_ERROR]Init OrcReader failed. reason = Failed to parse the postscript from hdfs://xxx/user/hive/warehouse/xxx.db/xxx/_temporary/0/_temporary/attempt_202311010952379045223848154629697_0013_m_000442_3355/part-00442-3f97ac18-0583-4d2c-8995-8d21ceda56ec-c000.snappy.orc
0. @ 0x000000000bdbd89f
doris::Status doris::Status::Error<6, true, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >&>(std::basic_string_view<char, std::char_traits<char> >, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >&) /var/local/ldb-toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/unique_ptr.h:173 in /hdpData10/yarn/localcache/usercache/hive/appcache/application_1696938614818_1832823/filecache/13/doris-be-36 [...]
doris::vectorized::OrcReader::_create_file_reader() /var/local/ldb-toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/basic_string.h:187 in /hdpData10/yarn/localcache/usercache/hive/appcache/application_1696938614818_1832823/filecache/13/doris-be-360-master.tar.gz/apache-doris-be-360-master-bin-x86_64/lib/doris_be 2. @ 0x000000000ecceff9
doris::vectorized::OrcReader::init_reader(std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const*, std::unordered_map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::variant<doris::ColumnValueRange<(doris::PrimitiveType)3>, doris::ColumnValueRange<(doris::PrimitiveType)4>, doris::ColumnValueRange<(doris: [...]
doris::vectorized::VFileScanner::_get_next_reader() /root/doris-master/be/src/vec/exec/scan/vfile_scanner.cpp:800 in /hdpData10/yarn/localcache/usercache/hive/appcache/application_1696938614818_1832823/filecache/13/doris-be-360-master.tar.gz/apache-doris-be-360-master-bin-x86_64/lib/doris_be 4. @ 0x000000000ff4256f
doris::vectorized::VFileScanner::_get_block_impl(doris::RuntimeState*, doris::vectorized::Block*, bool*) /root/doris-master/be/src/common/status.h:448 in /hdpData10/yarn/localcache/usercache/hive/appcache/application_1696938614818_1832823/filecache/13/doris-be-360-master.tar.gz/apache-doris-be-360-master-bin-x86_64/lib/doris_be 5. @ 0x000000000ffdb1aa
doris::vectorized::VScanner::get_block(doris::RuntimeState*, doris::vectorized::Block*, bool*) /root/doris-master/be/src/vec/exec/scan/vscanner.cpp:0 in /hdpData10/yarn/localcache/usercache/hive/appcache/application_1696938614818_1832823/filecache/13/doris-be-360-master.tar.gz/apache-doris-be-360-master-bin-x86_64/lib/doris_be 6. @ 0x000000000ff3b379
doris::vectorized::ScannerScheduler::_scanner_scan(doris::vectorized::ScannerScheduler*, doris::vectorized::ScannerContext*, std::shared_ptr<doris::vectorized::VScanner>) /root/doris-master/be/src/common/status.h:346 in /hdpData10/yarn/localcache/usercache/hive/appcache/application_1696938614818_1832823/filecache/13/doris-be-360-master.tar.gz/apache-doris-be-360-master-bin-x86_64/lib/doris_be 7. @ 0x000000000ff3ca11 std::_Function_handler<void (), doris::vectorized::ScannerSch [...]
doris::ThreadPool::dispatch_thread() /root/doris-master/be/src/util/threadpool.cpp:0 in /hdpData10/yarn/localcache/usercache/hive/appcache/application_1696938614818_1832823/filecache/13/doris-be-360-master.tar.gz/apache-doris-be-360-master-bin-x86_64/lib/doris_be 9. @ 0x000000000c97c2ac
doris::Thread::supervise_thread(void*) /var/local/ldb-toolchain/bin/../usr/include/pthread.h:562 in /hdpData10/yarn/localcache/usercache/hive/appcache/application_1696938614818_1832823/filecache/13/doris-be-360-master.tar.gz/apache-doris-be-360-master-bin-x86_64/lib/doris_be 10. @ 0x0000000000007ea5 start_thread in /usr/lib64/libpthread-2.17.so 11. @ 0x00000000000feb0d __clone in /usr/lib64/libc-2.17.so 0. @ 0x000000000c647366
doris::Status doris::Status::Error<6, true, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >(std::basic_string_view<char, std::char_traits<char> >, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >&&) /var/local/ldb-toolchain/bin/../lib/gcc/x86_6 [...]
doris::vectorized::VFileScanner::_get_next_reader() /var/local/ldb-toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/basic_string.h:187 in /hdpData10/yarn/localcache/usercache/hive/appcache/application_1696938614818_1832823/filecache/13/doris-be-360-master.tar.gz/apache-doris-be-360-master-bin-x86_64/lib/doris_be 2. @ 0x000000000ff4256f
doris::vectorized::VFileScanner::_get_block_impl(doris::RuntimeState*, doris::vectorized::Block*, bool*) /root/doris-master/be/src/common/status.h:448 in /hdpData10/yarn/localcache/usercache/hive/appcache/application_1696938614818_1832823/filecache/13/doris-be-360-master.tar.gz/apache-doris-be-360-master-bin-x86_64/lib/doris_be 3. @ 0x000000000ffdb1aa
doris::vectorized::VScanner::get_block(doris::RuntimeState*, doris::vectorized::Block*, bool*) /root/doris-master/be/src/vec/exec/scan/vscanner.cpp:0 in /hdpData10/yarn/localcache/usercache/hive/appcache/application_1696938614818_1832823/filecache/13/doris-be-360-master.tar.gz/apache-doris-be-360-master-bin-x86_64/lib/doris_be 4. @ 0x000000000ff3b379
doris::vectorized::ScannerScheduler::_scanner_scan(doris::vectorized::ScannerScheduler*, doris::vectorized::ScannerContext*, std::shared_ptr<doris::vectorized::VScanner>) /root/doris-master/be/src/common/status.h:346 in /hdpData10/yarn/localcache/usercache/hive/appcache/a…
```
Co-authored-by: wangxiangyu <wa...@360shuke.com>
---
.../java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java | 5 +++++
1 file changed, 5 insertions(+)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java
index 480edd60f2c..bac891eb920 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java
@@ -1058,9 +1058,14 @@ public class HiveMetaStoreCache {
return false;
}
for (String name : path.toString().split("/")) {
+ // generated by hive
if (name.startsWith(".hive-staging")) {
return false;
}
+ // generated by spark
+ if ("_temporary".equals(name)) {
+ return false;
+ }
}
return true;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org