[impala] 02/05: IMPALA-10504: Add tracing for remote block reads

[st...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

stigahuang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git

commit c8d94e6e365ca21d731617767bd7793f67f32222
Author: Kurt Deschler <kd...@cloudera.com>
AuthorDate: Sun Sep 27 16:37:32 2020 -0500

    IMPALA-10504: Add tracing for remote block reads
    
    This patch logs metadata for the first unexpected remote read of each
    scanrange when the flag fs_trace_remote_reads is set to true. This
    logging is intended to help diagnose the root cause of remote reads.
    
    Since a message may be logged for each scan range, there could be
    several hundred lines of output in a degenerate case. However, the
    remote read condition is not expected and verbose output may be needed
    to diagnose the root cause.
    
    Reviewed-by: Aman Sinha <am...@cloudera.com>
    Change-Id: I8c6a3e92f44813048022edf2b91299b3b0a20257
    Reviewed-on: http://gerrit.cloudera.org:8080/17062
    Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
    Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
 be/src/runtime/io/hdfs-file-reader.cc | 39 +++++++++++++++++++++++++++++++++++
 be/src/runtime/io/hdfs-file-reader.h  |  5 +++++
 2 files changed, 44 insertions(+)

diff --git a/be/src/runtime/io/hdfs-file-reader.cc b/be/src/runtime/io/hdfs-file-reader.cc
index 1441a2d..5683c5d 100644
--- a/be/src/runtime/io/hdfs-file-reader.cc
+++ b/be/src/runtime/io/hdfs-file-reader.cc
@@ -42,6 +42,9 @@ DEFINE_int64(fs_slow_read_log_threshold_ms, 10L * 1000L,
     "Log diagnostics about I/Os issued via the HDFS client that take longer than this "
     "threshold.");
 
+DEFINE_bool(fs_trace_remote_reads, false,
+    "(Advanced) Log block locations for remote reads.");
+
 #ifndef NDEBUG
 DECLARE_int32(stress_disk_read_delay_ms);
 #endif
@@ -78,6 +81,30 @@ Status HdfsFileReader::Open(bool use_file_handle_cache) {
   return Status::OK();
 }
 
+std::string HdfsFileReader::GetHostList(int64_t file_offset,
+    int64_t bytes_to_read) const {
+  char*** hosts = hdfsGetHosts(hdfs_fs_, scan_range_->file_string()->c_str(),
+      file_offset, bytes_to_read);
+  if (hosts) {
+    std::ostringstream ostr;
+    int blockIndex = 0;
+    while (hosts[blockIndex]) {
+      ostr << " [" << blockIndex << "] { ";
+      int hostIndex = 0;
+      while (hosts[blockIndex][hostIndex]) {
+        if(hostIndex > 0) ostr << ", ";
+        ostr << hosts[blockIndex][hostIndex];
+        hostIndex++;
+      }
+      ostr << " }";
+      blockIndex++;
+    }
+    hdfsFreeHosts(hosts);
+    return ostr.str();
+  }
+  return "";
+}
+
 Status HdfsFileReader::ReadFromPos(DiskQueue* queue, int64_t file_offset, uint8_t* buffer,
     int64_t bytes_to_read, int64_t* bytes_read, bool* eof) {
   DCHECK(scan_range_->read_in_flight());
@@ -209,8 +236,20 @@ Status HdfsFileReader::ReadFromPos(DiskQueue* queue, int64_t file_offset, uint8_
       }
       *bytes_read += current_bytes_read;
 
+      bool is_first_read = (num_remote_bytes_ == 0);
       // Collect and accumulate statistics
       GetHdfsStatistics(hdfs_file, log_slow_read);
+      if (FLAGS_fs_trace_remote_reads && expected_local_ &&
+          num_remote_bytes_ > 0 && is_first_read) {
+        // Only log the first unexpected remote read for scan range
+        LOG(INFO)
+            << "First remote read of scan range on file "
+            << *scan_range_->file_string()
+            << " " << num_remote_bytes_
+            << " bytes. offsets " << file_offset
+            << "-" << file_offset+bytes_to_read-1
+            << GetHostList(file_offset, bytes_to_read);
+      }
     }
 
     int64_t cached_bytes_missed = *bytes_read - cached_read;
diff --git a/be/src/runtime/io/hdfs-file-reader.h b/be/src/runtime/io/hdfs-file-reader.h
index 29f0e2c..ed8acc7 100644
--- a/be/src/runtime/io/hdfs-file-reader.h
+++ b/be/src/runtime/io/hdfs-file-reader.h
@@ -78,6 +78,11 @@ private:
   /// true, the statistics are logged.
   void GetHdfsStatistics(hdfsFile hdfs_file, bool log_stats);
 
+  /// Return a string that contains the block indexes and list of hosts where
+  /// each block resides. i.e. [0] { hdfshost1, hdfshost2, hdfshost3 }
+
+  std::string GetHostList(int64_t file_offset, int64_t bytes_to_read) const;
+
   /// Hadoop filesystem that contains the file being read.
   hdfsFS const hdfs_fs_;