You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mo...@apache.org on 2021/11/11 07:21:32 UTC
[incubator-doris] branch master updated: [libhdfs] Add errno for
hdfs writer. when no dir, hdfs writer open failed,
the dir need to be created. (#7050)
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
The following commit(s) were added to refs/heads/master by this push:
new 632f8fc [libhdfs] Add errno for hdfs writer. when no dir, hdfs writer open failed, the dir need to be created. (#7050)
632f8fc is described below
commit 632f8fcc754d23f5674e32ddef2ddec9664b759e
Author: pengxiangyu <di...@163.com>
AuthorDate: Thu Nov 11 15:21:21 2021 +0800
[libhdfs] Add errno for hdfs writer. when no dir, hdfs writer open failed, the dir need to be created. (#7050)
1. Add errno message for hdfs writer failed.
2. When call openWrite for hdfs, the dir will be created when it doesn't exist,
---
be/src/exec/hdfs_file_reader.cpp | 16 ++++++++++------
be/src/exec/hdfs_writer.cpp | 32 +++++++++++++++++++++++++++++---
be/src/exprs/bitmap_function.h | 1 +
be/src/util/bitmap_value.h | 2 +-
4 files changed, 41 insertions(+), 10 deletions(-)
diff --git a/be/src/exec/hdfs_file_reader.cpp b/be/src/exec/hdfs_file_reader.cpp
index a6f509f..0ba1a3a 100644
--- a/be/src/exec/hdfs_file_reader.cpp
+++ b/be/src/exec/hdfs_file_reader.cpp
@@ -20,6 +20,7 @@
#include <unistd.h>
#include "common/logging.h"
+#include "service/backend_options.h"
namespace doris {
@@ -80,7 +81,8 @@ Status HdfsFileReader::open() {
_hdfs_file = hdfsOpenFile(_hdfs_fs, _path.c_str(), O_RDONLY, 0, 0, 0);
if (_hdfs_file == nullptr) {
std::stringstream ss;
- ss << "open file failed. " << _namenode << _path;
+ ss << "open file failed. " << "(BE: " << BackendOptions::get_localhost() << ")"
+ << _namenode << _path << ", err: " << strerror(errno);;
return Status::InternalError(ss.str());
}
LOG(INFO) << "open file. " << _namenode << _path;
@@ -139,7 +141,8 @@ Status HdfsFileReader::readat(int64_t position, int64_t nbytes, int64_t* bytes_r
int ret = hdfsSeek(_hdfs_fs, _hdfs_file, position);
if (ret != 0) { // check fseek return value
std::stringstream ss;
- ss << "hdfsSeek failed. " << _namenode << _path;
+ ss << "hdfsSeek failed. " << "(BE: " << BackendOptions::get_localhost() << ")"
+ << _namenode << _path << ", err: " << strerror(errno);;
return Status::InternalError(ss.str());
}
}
@@ -147,7 +150,8 @@ Status HdfsFileReader::readat(int64_t position, int64_t nbytes, int64_t* bytes_r
*bytes_read = hdfsRead(_hdfs_fs, _hdfs_file, out, nbytes);
if (*bytes_read < 0) {
std::stringstream ss;
- ss << "Read hdfs file failed. " << _namenode << _path;
+ ss << "Read hdfs file failed. " << "(BE: " << BackendOptions::get_localhost() << ")"
+ << _namenode << _path << ", err: " << strerror(errno);;
return Status::InternalError(ss.str());
}
_current_offset += *bytes_read; // save offset with file
@@ -165,7 +169,7 @@ int64_t HdfsFileReader::size() {
}
hdfsFileInfo* file_info = hdfsGetPathInfo(_hdfs_fs, _path.c_str());
if (file_info == nullptr) {
- LOG(WARNING) << "get path info failed: " << _namenode << _path;
+ LOG(WARNING) << "get path info failed: " << _namenode << _path << ", err: " << strerror(errno);;
close();
return -1;
}
@@ -183,8 +187,8 @@ Status HdfsFileReader::seek(int64_t position) {
if (res != 0) {
char err_buf[64];
std::stringstream ss;
- ss << "Seek to offset failed. offset=" << position
- << ", error=" << strerror_r(errno, err_buf, 64);
+ ss << "Seek to offset failed. " << "(BE: " << BackendOptions::get_localhost() << ")"
+ << " offset=" << position << ", err: " << strerror(errno);
return Status::InternalError(ss.str());
}
return Status::OK();
diff --git a/be/src/exec/hdfs_writer.cpp b/be/src/exec/hdfs_writer.cpp
index a00413f..d608a77 100644
--- a/be/src/exec/hdfs_writer.cpp
+++ b/be/src/exec/hdfs_writer.cpp
@@ -17,7 +17,10 @@
#include "exec/hdfs_writer.h"
+#include <filesystem>
+
#include "common/logging.h"
+#include "service/backend_options.h"
namespace doris {
const static std::string FS_KEY = "fs.defaultFS";
@@ -47,11 +50,30 @@ Status HDFSWriter::open() {
// the path already exists
return Status::AlreadyExist(_path + " already exists.");
}
+
+ std::filesystem::path hdfs_path(_path);
+ std::string hdfs_dir = hdfs_path.parent_path().string();
+ exists = hdfsExists(_hdfs_fs, hdfs_dir.c_str());
+ if (exists != 0) {
+ LOG(INFO) << "hdfs dir doesn't exist, create it: " << hdfs_dir;
+ int ret = hdfsCreateDirectory(_hdfs_fs, hdfs_dir.c_str());
+ if (ret != 0) {
+ std::stringstream ss;
+ ss << "create dir failed. " << "(BE: " << BackendOptions::get_localhost() << ")"
+ << " namenode: " << _namenode << " path: " << hdfs_dir
+ << ", err: " << strerror(errno);
+ LOG(WARNING) << ss.str();
+ return Status::InternalError(ss.str());
+ }
+ }
// open file
_hdfs_file = hdfsOpenFile(_hdfs_fs, _path.c_str(), O_WRONLY, 0, 0, 0);
if (_hdfs_file == nullptr) {
std::stringstream ss;
- ss << "open file failed. namenode:" << _namenode << " path:" << _path;
+ ss << "open file failed. " << "(BE: " << BackendOptions::get_localhost() << ")"
+ << " namenode:" << _namenode << " path:" << _path
+ << ", err: " << strerror(errno);
+ LOG(WARNING) << ss.str();
return Status::InternalError(ss.str());
}
LOG(INFO) << "open file. namenode:" << _namenode << " path:" << _path;
@@ -66,7 +88,9 @@ Status HDFSWriter::write(const uint8_t* buf, size_t buf_len, size_t* written_len
int32_t result = hdfsWrite(_hdfs_fs, _hdfs_file, buf, buf_len);
if (result < 0) {
std::stringstream ss;
- ss << "write file failed. namenode:" << _namenode << " path:" << _path;
+ ss << "write file failed. " << "(BE: " << BackendOptions::get_localhost() << ")"
+ << "namenode:" << _namenode << " path:" << _path
+ << ", err: " << strerror(errno);
LOG(WARNING) << ss.str();
return Status::InternalError(ss.str());
}
@@ -91,7 +115,9 @@ Status HDFSWriter::close() {
int result = hdfsFlush(_hdfs_fs, _hdfs_file);
if (result == -1) {
std::stringstream ss;
- ss << "failed to flush hdfs file. namenode:" << _namenode << " path:" << _path;
+ ss << "failed to flush hdfs file. " << "(BE: " << BackendOptions::get_localhost() << ")"
+ << "namenode:" << _namenode << " path:" << _path
+ << ", err: " << strerror(errno);
LOG(WARNING) << ss.str();
return Status::InternalError(ss.str());
}
diff --git a/be/src/exprs/bitmap_function.h b/be/src/exprs/bitmap_function.h
index 363b9b7..2cba685 100644
--- a/be/src/exprs/bitmap_function.h
+++ b/be/src/exprs/bitmap_function.h
@@ -110,6 +110,7 @@ public:
const BigIntVal& range_start, const BigIntVal& cardinality_limit);
static StringVal sub_bitmap(FunctionContext* ctx, const StringVal& src,
const BigIntVal& offset, const BigIntVal& cardinality_limit);
+
};
} // namespace doris
#endif //DORIS_BE_SRC_QUERY_EXPRS_BITMAP_FUNCTION_H
diff --git a/be/src/util/bitmap_value.h b/be/src/util/bitmap_value.h
index 6d3710f..34f0aba 100644
--- a/be/src/util/bitmap_value.h
+++ b/be/src/util/bitmap_value.h
@@ -1489,7 +1489,7 @@ public:
break;
}
}
- return count;
+ return count;
}
/**
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org