[GitHub] [arrow] LinGeLin commented on issue #13403: [C++] How to speed up arrow's reading of S3 Parquet files？

[GitBox <gi...@apache.org>]

LinGeLin commented on issue #13403:
URL: https://github.com/apache/arrow/issues/13403#issuecomment-1206001646

   Thanks, I made a simple mistake. I used the Debug version to test, and of course it was slow. The parameters you mentioned might be useful, but now I read S3 parquet in a different way, because the dataset always core dump when the program exits
   `
         Status ReadFile(int file_index, bool background = false)
             TF_EXCLUSIVE_LOCKS_REQUIRED(mu_) {
           auto access_file =
               s3fs_->OpenInputFile(dataset()->parquet_files_[file_index])
                   .ValueOrDie();
   
           parquet::ArrowReaderProperties properties;
           properties.set_use_threads(true);
           properties.set_pre_buffer(true);
           parquet::ReaderProperties parquet_properties =
               parquet::default_reader_properties();
   
           std::shared_ptr<parquet::arrow::FileReaderBuilder> builder =
               std::make_shared<parquet::arrow::FileReaderBuilder>();
           builder->Open(access_file, parquet_properties);
   
           std::unique_ptr<parquet::arrow::FileReader> reader;
           builder->properties(properties)->Build(&reader);
   
           if (column_indices_.empty()) {
             std::shared_ptr<arrow::Schema> schema;
             reader->GetSchema(&schema);
             // check column name exist
             std::string err_column_names;
             for (const auto& name : dataset()->column_names_) {
               int fieldIndex = schema->GetFieldIndex(name);
               column_indices_.push_back(fieldIndex);
               if (-1 == fieldIndex) {
                 err_column_names = err_column_names + " " + name;
               }
             }
   
             if (err_column_names.length() != 0) {
               return errors::InvalidArgument("these column names don't exist: ",
                                              err_column_names);
             }
           }
           // Read file columns and build a table
           std::shared_ptr<::arrow::Table> table;
           CHECK_ARROW(reader->ReadTable(column_indices_, &table));
           // Convert the table to a sequence of batches
           std::shared_ptr<arrow::RecordBatchReader> batch_reader =
               std::make_shared<arrow::TableBatchReader>(table);
           std::shared_ptr<arrow::RecordBatch> batch = nullptr;
   
           // filter
           if (!dataset()->filter_.empty()) {
             auto scanner_builder =
                 arrow::dataset::ScannerBuilder::FromRecordBatchReader(
                     batch_reader);
             arrow::compute::Expression filter_expr;
             TF_RETURN_IF_ERROR(
                 ArrowUtil::ParseExpression(dataset()->filter_, filter_expr));
             scanner_builder->Filter(filter_expr);
             auto scanner = scanner_builder->Finish().ValueOrDie();
             batch_reader = scanner->ToRecordBatchReader().ValueOrDie();
           }
   
           CHECK_ARROW(batch_reader->ReadNext(&batch));
           TF_RETURN_IF_ERROR(CheckBatchColumnTypes(batch));
           next_record_batches_.clear();
           while (batch != nullptr) {
             if (!background) {
               record_batches_.emplace_back(batch);
             } else {
               next_record_batches_.emplace_back(batch);
             }
             CHECK_ARROW(batch_reader->ReadNext(&batch));
           }
   
           if (background) {
             mutex_lock lk(cv_mu_);
             background_thread_finished_ = true;
             cv_.notify_all();
           }
   
           return Status::OK();
         }
   `
   
   Any other suggestions are also fine


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: github-unsubscribe@arrow.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org