You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mo...@apache.org on 2021/11/12 03:47:06 UTC
[incubator-doris] 01/09: [libhdfs] Add errno for hdfs writer. when
no dir, hdfs writer open failed, the dir need to be created. (#7050)
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-0.15
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
commit cecd369f3f463aebe295d8b233d319aeb66756c4
Author: pengxiangyu <di...@163.com>
AuthorDate: Thu Nov 11 15:21:21 2021 +0800
[libhdfs] Add errno for hdfs writer. when no dir, hdfs writer open failed, the dir need to be created. (#7050)
1. Add errno message for hdfs writer failed.
2. When call openWrite for hdfs, the dir will be created when it doesn't exist,
---
be/src/exec/hdfs_file_reader.cpp | 16 ++++++++++------
be/src/exec/hdfs_writer.cpp | 32 +++++++++++++++++++++++++++++---
2 files changed, 39 insertions(+), 9 deletions(-)
diff --git a/be/src/exec/hdfs_file_reader.cpp b/be/src/exec/hdfs_file_reader.cpp
index a6f509f..0ba1a3a 100644
--- a/be/src/exec/hdfs_file_reader.cpp
+++ b/be/src/exec/hdfs_file_reader.cpp
@@ -20,6 +20,7 @@
#include <unistd.h>
#include "common/logging.h"
+#include "service/backend_options.h"
namespace doris {
@@ -80,7 +81,8 @@ Status HdfsFileReader::open() {
_hdfs_file = hdfsOpenFile(_hdfs_fs, _path.c_str(), O_RDONLY, 0, 0, 0);
if (_hdfs_file == nullptr) {
std::stringstream ss;
- ss << "open file failed. " << _namenode << _path;
+ ss << "open file failed. " << "(BE: " << BackendOptions::get_localhost() << ")"
+ << _namenode << _path << ", err: " << strerror(errno);;
return Status::InternalError(ss.str());
}
LOG(INFO) << "open file. " << _namenode << _path;
@@ -139,7 +141,8 @@ Status HdfsFileReader::readat(int64_t position, int64_t nbytes, int64_t* bytes_r
int ret = hdfsSeek(_hdfs_fs, _hdfs_file, position);
if (ret != 0) { // check fseek return value
std::stringstream ss;
- ss << "hdfsSeek failed. " << _namenode << _path;
+ ss << "hdfsSeek failed. " << "(BE: " << BackendOptions::get_localhost() << ")"
+ << _namenode << _path << ", err: " << strerror(errno);;
return Status::InternalError(ss.str());
}
}
@@ -147,7 +150,8 @@ Status HdfsFileReader::readat(int64_t position, int64_t nbytes, int64_t* bytes_r
*bytes_read = hdfsRead(_hdfs_fs, _hdfs_file, out, nbytes);
if (*bytes_read < 0) {
std::stringstream ss;
- ss << "Read hdfs file failed. " << _namenode << _path;
+ ss << "Read hdfs file failed. " << "(BE: " << BackendOptions::get_localhost() << ")"
+ << _namenode << _path << ", err: " << strerror(errno);;
return Status::InternalError(ss.str());
}
_current_offset += *bytes_read; // save offset with file
@@ -165,7 +169,7 @@ int64_t HdfsFileReader::size() {
}
hdfsFileInfo* file_info = hdfsGetPathInfo(_hdfs_fs, _path.c_str());
if (file_info == nullptr) {
- LOG(WARNING) << "get path info failed: " << _namenode << _path;
+ LOG(WARNING) << "get path info failed: " << _namenode << _path << ", err: " << strerror(errno);;
close();
return -1;
}
@@ -183,8 +187,8 @@ Status HdfsFileReader::seek(int64_t position) {
if (res != 0) {
char err_buf[64];
std::stringstream ss;
- ss << "Seek to offset failed. offset=" << position
- << ", error=" << strerror_r(errno, err_buf, 64);
+ ss << "Seek to offset failed. " << "(BE: " << BackendOptions::get_localhost() << ")"
+ << " offset=" << position << ", err: " << strerror(errno);
return Status::InternalError(ss.str());
}
return Status::OK();
diff --git a/be/src/exec/hdfs_writer.cpp b/be/src/exec/hdfs_writer.cpp
index a00413f..d608a77 100644
--- a/be/src/exec/hdfs_writer.cpp
+++ b/be/src/exec/hdfs_writer.cpp
@@ -17,7 +17,10 @@
#include "exec/hdfs_writer.h"
+#include <filesystem>
+
#include "common/logging.h"
+#include "service/backend_options.h"
namespace doris {
const static std::string FS_KEY = "fs.defaultFS";
@@ -47,11 +50,30 @@ Status HDFSWriter::open() {
// the path already exists
return Status::AlreadyExist(_path + " already exists.");
}
+
+ std::filesystem::path hdfs_path(_path);
+ std::string hdfs_dir = hdfs_path.parent_path().string();
+ exists = hdfsExists(_hdfs_fs, hdfs_dir.c_str());
+ if (exists != 0) {
+ LOG(INFO) << "hdfs dir doesn't exist, create it: " << hdfs_dir;
+ int ret = hdfsCreateDirectory(_hdfs_fs, hdfs_dir.c_str());
+ if (ret != 0) {
+ std::stringstream ss;
+ ss << "create dir failed. " << "(BE: " << BackendOptions::get_localhost() << ")"
+ << " namenode: " << _namenode << " path: " << hdfs_dir
+ << ", err: " << strerror(errno);
+ LOG(WARNING) << ss.str();
+ return Status::InternalError(ss.str());
+ }
+ }
// open file
_hdfs_file = hdfsOpenFile(_hdfs_fs, _path.c_str(), O_WRONLY, 0, 0, 0);
if (_hdfs_file == nullptr) {
std::stringstream ss;
- ss << "open file failed. namenode:" << _namenode << " path:" << _path;
+ ss << "open file failed. " << "(BE: " << BackendOptions::get_localhost() << ")"
+ << " namenode:" << _namenode << " path:" << _path
+ << ", err: " << strerror(errno);
+ LOG(WARNING) << ss.str();
return Status::InternalError(ss.str());
}
LOG(INFO) << "open file. namenode:" << _namenode << " path:" << _path;
@@ -66,7 +88,9 @@ Status HDFSWriter::write(const uint8_t* buf, size_t buf_len, size_t* written_len
int32_t result = hdfsWrite(_hdfs_fs, _hdfs_file, buf, buf_len);
if (result < 0) {
std::stringstream ss;
- ss << "write file failed. namenode:" << _namenode << " path:" << _path;
+ ss << "write file failed. " << "(BE: " << BackendOptions::get_localhost() << ")"
+ << "namenode:" << _namenode << " path:" << _path
+ << ", err: " << strerror(errno);
LOG(WARNING) << ss.str();
return Status::InternalError(ss.str());
}
@@ -91,7 +115,9 @@ Status HDFSWriter::close() {
int result = hdfsFlush(_hdfs_fs, _hdfs_file);
if (result == -1) {
std::stringstream ss;
- ss << "failed to flush hdfs file. namenode:" << _namenode << " path:" << _path;
+ ss << "failed to flush hdfs file. " << "(BE: " << BackendOptions::get_localhost() << ")"
+ << "namenode:" << _namenode << " path:" << _path
+ << ", err: " << strerror(errno);
LOG(WARNING) << ss.str();
return Status::InternalError(ss.str());
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org