You are viewing a plain text version of this content. The canonical link for it is here.
Posted to user@arrow.apache.org by 1057445597 <10...@qq.com> on 2022/07/01 07:39:28 UTC

why set use thread doesn't work?

Setting set_use_thread(true) is useful when reading local files, but not when reading S3, That's a 10-fold difference, is there any way to improve the speed of reading S3?


// EnsureS3Initialized();
// S3Options s3Options = S3Options::FromAccessKey(K_ACCESS_KEY1, K_SECRET_KEY1);
// s3Options.endpoint_override = K_ENDPOINT_OVERRIDE1;
// std::shared_ptr<S3FileSystem&gt; s3fs = S3FileSystem::Make(s3Options).ValueOrDie();
// auto infile = s3fs-&gt;OpenInputFile(path).ValueOrDie();


  std::shared_ptr<arrow::io::ReadableFile&gt; infile;
  PARQUET_ASSIGN_OR_THROW(infile,
                          arrow::io::ReadableFile::Open(path,
                                                        arrow::default_memory_pool()));

  std::unique_ptr<parquet::arrow::FileReader&gt; reader;
  PARQUET_THROW_NOT_OK(
      parquet::arrow::OpenFile(infile, arrow::default_memory_pool(), &amp;reader));
  reader-&gt;set_use_threads(false);
  // reader-&gt;set_use_threads(true);
  std::shared_ptr<arrow::Table&gt; table;






1057445597
1057445597@qq.com



&nbsp;